From 717c6c3c11e59e7b15991d43d21c1c8173268caa Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 30 Jul 2026 13:27:43 +0800 Subject: [PATCH 01/10] [enhancement](scan) optimize parquet v2 predicate filtering Keep identity selections implicit, refresh late predicates at row-group boundaries, and reuse dictionary predicate bitmaps within a query. --- be/src/exec/scan/file_scanner_v2.cpp | 7 + be/src/exec/scan/file_scanner_v2.h | 1 + be/src/format_v2/column_mapper.cpp | 9 +- be/src/format_v2/column_mapper.h | 9 +- be/src/format_v2/file_reader.h | 9 + be/src/format_v2/parquet/parquet_profile.cpp | 6 + be/src/format_v2/parquet/parquet_profile.h | 6 +- be/src/format_v2/parquet/parquet_reader.cpp | 17 +- be/src/format_v2/parquet/parquet_reader.h | 4 + be/src/format_v2/parquet/parquet_scan.cpp | 171 ++++++++++++++---- be/src/format_v2/parquet/parquet_scan.h | 12 +- .../parquet/reader/native_column_reader.cpp | 2 - be/src/format_v2/parquet/selection_vector.h | 82 ++++++++- be/src/format_v2/table_reader.cpp | 44 +++++ be/src/format_v2/table_reader.h | 12 +- be/src/runtime/query_context.cpp | 8 + be/src/runtime/query_context.h | 5 + .../runtime/query_dictionary_filter_cache.h | 102 +++++++++++ .../parquet/parquet_reader_control_test.cpp | 52 ++++++ .../format_v2/parquet/parquet_scan_test.cpp | 2 + be/test/format_v2/table_reader_test.cpp | 59 ++++++ 21 files changed, 564 insertions(+), 55 deletions(-) create mode 100644 be/src/runtime/query_dictionary_filter_cache.h diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index c568832d67cddd..39dc99948734b8 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -442,6 +442,12 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } { + if (_table_reader_rf_num != _applied_rf_num) { + VExprContextSPtrs refreshed_conjuncts; + RETURN_IF_ERROR(_build_table_conjuncts(&refreshed_conjuncts)); + RETURN_IF_ERROR(_table_reader->refresh_conjuncts(std::move(refreshed_conjuncts))); + _table_reader_rf_num = _applied_rf_num; + } if (_should_run_adaptive_batch_size()) { _table_reader->set_batch_size(_predict_reader_batch_rows()); } @@ -549,6 +555,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { } COUNTER_UPDATE(_file_counter, 1); _has_prepared_split = true; + _table_reader_rf_num = _applied_rf_num; *eos = false; return Status::OK(); } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 658ec44ad7f051..87a68e2ab6d176 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -178,6 +178,7 @@ class FileScannerV2 final : public Scanner { std::shared_ptr _split_source; bool _first_scan_range = false; bool _has_prepared_split = false; + int _table_reader_rf_num = 0; TFileRangeDesc _current_range; std::string _current_range_path; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index c3dbe3fa9e7766..7e028370221c38 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2161,13 +2161,20 @@ Status TableColumnMapper::_build_filter_entries(const FileScanRequest& file_requ Status TableColumnMapper::create_scan_request( const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, + const std::map* fixed_local_positions) { // FileReader evaluates expressions against a file-local block. This mapper owns the // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); file_request->non_predicate_columns.clear(); file_request->predicate_only_columns.clear(); file_request->local_positions.clear(); + if (fixed_local_positions != nullptr) { + // A refreshed predicate may promote a lazy column, but the active split's block slots are + // immutable. Seed their positions before rebuilding expressions so every rewritten SlotRef + // continues to address the same physical column. + file_request->local_positions = *fixed_local_positions; + } file_request->conjuncts.clear(); file_request->delete_conjuncts.clear(); _filter_entries.clear(); diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index e03f836061ebf5..68d0ff357c0e52 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -193,10 +193,11 @@ class TableColumnMapper { // Convert a table-level scan request into a file-local scan request. table_filters preserve // row-level filtering semantics and are rewritten as file-local conjuncts. File-layer pruning // such as ZoneMap, dictionary, and bloom filter derives from those localized VExpr conjuncts. - virtual Status create_scan_request(const std::vector& table_filters, - const std::vector& projected_columns, - FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + virtual Status create_scan_request( + const std::vector& table_filters, + const std::vector& projected_columns, FileScanRequest* file_request, + RuntimeState* runtime_state = nullptr, + const std::map* fixed_local_positions = nullptr); // Localize table-level filters to the file schema. // Trivial mappings can copy structured predicates directly. Type changes may be localized with diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 3ff512975d2dcd..65a2d03417c605 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -305,6 +305,15 @@ class FileReader { return Status::OK(); } + // Readers opt in only when they can keep an immutable request for the active physical + // granule and switch a newer snapshot at a well-defined boundary. + virtual bool supports_scan_request_refresh() const { return false; } + + virtual Status queue_scan_request(std::shared_ptr request) { + (void)request; + return Status::NotSupported("FileReader does not support scan request refresh"); + } + virtual Status get_block(Block* file_block, size_t* rows, bool* eof) { if (rows != nullptr) { *rows = 0; diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 02b64d7b63c867..84eb6e0c700066 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -219,6 +219,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { profile, "DictFilterUnsupportedColumns", TUnit::UNIT, parquet_profile, 1); dict_filter_read_failures = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DictFilterReadFailures", TUnit::UNIT, parquet_profile, 1); + query_dict_filter_cache_hits = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "QueryDictionaryFilterCacheHits", TUnit::UNIT, parquet_profile, 1); + query_dict_filter_cache_misses = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "QueryDictionaryFilterCacheMisses", TUnit::UNIT, parquet_profile, 1); rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", TUnit::UNIT, parquet_profile, 1); bloom_filter_read_time = @@ -363,6 +367,8 @@ ParquetScanProfile ParquetProfile::scan_profile() const { dict_filter_vectorized_runtime_filter_columns, .dict_filter_unsupported_columns = dict_filter_unsupported_columns, .dict_filter_read_failures = dict_filter_read_failures, + .query_dict_filter_cache_hits = query_dict_filter_cache_hits, + .query_dict_filter_cache_misses = query_dict_filter_cache_misses, .rows_filtered_by_dict_filter = rows_filtered_by_dict_filter, .column_reader_profile = column_reader_profile(), }; diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 2282a70db30540..eb8524430d3c77 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -115,7 +115,9 @@ struct ParquetScanProfile { nullptr; // vectorized runtime-filter columns RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; // unsupported columns RuntimeProfile::Counter* dict_filter_read_failures = nullptr; // dictionary read failures - RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; // rows filtered by dict + RuntimeProfile::Counter* query_dict_filter_cache_hits = nullptr; + RuntimeProfile::Counter* query_dict_filter_cache_misses = nullptr; + RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; // rows filtered by dict ParquetColumnReaderProfile column_reader_profile; // nested column read statistics }; @@ -240,6 +242,8 @@ struct ParquetProfile { RuntimeProfile::Counter* dict_filter_vectorized_runtime_filter_columns = nullptr; RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; RuntimeProfile::Counter* dict_filter_read_failures = nullptr; + RuntimeProfile::Counter* query_dict_filter_cache_hits = nullptr; + RuntimeProfile::Counter* query_dict_filter_cache_misses = nullptr; RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; RuntimeProfile::Counter* bloom_filter_read_time = nullptr; }; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 4a80bc0403df8d..70c0efb5f37548 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -471,10 +471,22 @@ Status ParquetReader::open(std::shared_ptr request) { _state->scheduler.set_global_rowid_context(_global_rowid_context); _state->scheduler.set_scan_profile(_parquet_profile.scan_profile()); _state->scheduler.set_plan(std::move(row_group_plan)); + _state->scheduler.set_scan_request(request_snapshot); _eof = _state->scheduler.empty(); return Status::OK(); } +Status ParquetReader::queue_scan_request(std::shared_ptr request) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { + return Status::Uninitialized("ParquetReader is not open"); + } + DORIS_CHECK(request != nullptr); + RETURN_IF_ERROR(validate_requested_columns_supported(_state->file_schema, *request)); + _state->scheduler.queue_scan_request(request); + _request = std::move(request); + return Status::OK(); +} + Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { SCOPED_TIMER(_parquet_profile.total_time); if (_state == nullptr || _state->file_context.native_metadata == nullptr) { @@ -489,15 +501,14 @@ Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { *eof = true; return Status::OK(); } - auto request_snapshot = _request; - if (request_snapshot == nullptr) { + if (_request == nullptr) { return Status::Cancelled("ParquetReader is closed"); } const auto predicate_filtered_rows_before = _state->scheduler.predicate_filtered_rows(); const auto raw_rows_read_before = _state->scheduler.raw_rows_read(); Status st = _state->scheduler.read_next_batch(_state->file_context, _state->file_schema, - *request_snapshot, file_block, rows, eof); + file_block, rows, eof); if (!st.ok()) { if (_io_ctx != nullptr && _io_ctx->should_stop) { *rows = 0; diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index da6135b81ae838..fe95b93a9e0101 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -60,6 +60,10 @@ class ParquetReader : public format::FileReader { Status open(std::shared_ptr request) override; + bool supports_scan_request_refresh() const override { return true; } + + Status queue_scan_request(std::shared_ptr request) override; + Status get_block(Block* file_block, size_t* rows, bool* eof) override; Status get_aggregate_result(const format::FileAggregateRequest& request, diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 33ef6a08fedb2c..478735af714d6a 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -35,6 +35,7 @@ #include "core/column/column_nullable.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "exec/common/sip_hash.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vcompound_pred.h" #include "exprs/vectorized_fn_call.h" @@ -46,11 +47,23 @@ #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" +#include "runtime/query_context.h" +#include "runtime/query_dictionary_filter_cache.h" +#include "runtime/runtime_state.h" #include "util/defer_op.h" +#include "util/hash_util.hpp" #include "util/time.h" namespace doris::format::parquet { +void ParquetScanScheduler::set_runtime_state(RuntimeState* runtime_state) { + _runtime_state = runtime_state; + _query_dictionary_filter_cache = + runtime_state == nullptr || runtime_state->get_query_ctx() == nullptr + ? nullptr + : &runtime_state->get_query_ctx()->query_dictionary_filter_cache(); +} + namespace detail { std::vector order_adaptive_predicates( @@ -587,14 +600,7 @@ void update_counter_if_not_null(RuntimeProfile::Counter* counter, int64_t value) uint16_t apply_filter_to_selection(const IColumn::Filter& filter, SelectionVector* selection, uint16_t selected_rows) { - uint16_t new_selected_rows = 0; - for (uint16_t selection_idx = 0; selection_idx < selected_rows; ++selection_idx) { - const auto row_idx = selection->get_index(selection_idx); - if (filter[row_idx] != 0) { - selection->set_index(new_selected_rows++, static_cast(row_idx)); - } - } - return new_selected_rows; + return cast_set(selection->compact_with_row_filter(filter.data(), selected_rows)); } Status execute_compact_filter_conjuncts(const VExprContextSPtrs& conjuncts, size_t rows, @@ -752,14 +758,8 @@ uint16_t apply_compact_filter_to_selection(const IColumn::Filter& filter, SelectionVector* selection, uint16_t selected_rows) { DORIS_CHECK(selection != nullptr); DORIS_CHECK(filter.size() == selected_rows); - uint16_t new_selected_rows = 0; - for (uint16_t selection_idx = 0; selection_idx < selected_rows; ++selection_idx) { - if (filter[selection_idx] != 0) { - selection->set_index(new_selected_rows++, static_cast( - selection->get_index(selection_idx))); - } - } - return new_selected_rows; + return cast_set( + selection->compact_with_selection_filter(filter.data(), selected_rows)); } IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, @@ -908,6 +908,29 @@ void ParquetScanScheduler::reset() { reset_current_row_group(); } +void ParquetScanScheduler::set_scan_request(std::shared_ptr request) { + DORIS_CHECK(request != nullptr); + _active_request = std::move(request); + _pending_request.reset(); + _predicate_schedule_request = nullptr; +} + +void ParquetScanScheduler::queue_scan_request(std::shared_ptr request) { + DORIS_CHECK(request != nullptr); + _pending_request = std::move(request); +} + +void ParquetScanScheduler::activate_pending_scan_request_at_row_group_boundary() { + if (_has_current_row_group || !_pending_predicate_selection.empty() || + _pending_request == nullptr) { + return; + } + // Column readers and predicate schedules retain request-derived state for one row group. Swap + // only after they are gone; the refreshed request may promote a lazy column to a predicate. + _active_request = std::move(_pending_request); + _predicate_schedule_request = nullptr; +} + void ParquetScanScheduler::reset_current_row_group() { // RuntimeProfile updates are amortized on the batch path, but a row-group transition destroys // the reader tree. Force the final delta out before clearing it so short row groups and early @@ -1677,6 +1700,63 @@ Status build_dictionary_entry_filter(size_t block_position, return Status::OK(); } +bool contains_mutable_topn_filter(const VExprSPtr& expression) { + if (expression == nullptr) { + return false; + } + if (expression->is_topn_filter()) { + return true; + } + return std::ranges::any_of(expression->children(), contains_mutable_topn_filter); +} + +std::optional dictionary_predicate_digest(size_t block_position, + const VExprContextSPtrs& conjuncts) { + uint64_t digest = 0x6a09e667f3bcc909ULL; + digest = HashUtil::hash64(&block_position, sizeof(block_position), digest); + for (const auto& conjunct : conjuncts) { + if (conjunct == nullptr || contains_mutable_topn_filter(conjunct->root())) { + // TopN's bound tightens in place and is intentionally absent from VExpr::get_digest(). + // Reusing an older bitmap would therefore admit rows beyond the current frontier. + return std::nullopt; + } + digest = conjunct->get_digest(digest); + if (digest == 0) { + return std::nullopt; + } + } + return digest; +} + +std::optional dictionary_filter_cache_key( + size_t block_position, const ParquetColumnSchema& column_schema, + const VExprContextSPtrs& conjuncts, const IColumn& dictionary) { + const auto expression_digest = dictionary_predicate_digest(block_position, conjuncts); + if (!expression_digest.has_value() || + dictionary.size() > std::numeric_limits::max()) { + return std::nullopt; + } + const auto primitive_type = remove_nullable(column_schema.type)->get_primitive_type(); + SipHash dictionary_hash; + const auto type_name = remove_nullable(column_schema.type)->get_name(); + dictionary_hash.update(type_name.data(), type_name.size()); + const uint64_t entries = dictionary.size(); + dictionary_hash.update(entries); + for (size_t entry = 0; entry < dictionary.size(); ++entry) { + dictionary.update_hash_with_value(entry, dictionary_hash); + } + uint64_t hash_low = 0; + uint64_t hash_high = 0; + dictionary_hash.get128(hash_low, hash_high); + return QueryDictionaryFilterCacheKey { + .expression_digest = *expression_digest, + .dictionary_hash_low = hash_low, + .dictionary_hash_high = hash_high, + .dictionary_entries = static_cast(dictionary.size()), + .primitive_type = primitive_type, + }; +} + } // namespace Status ParquetScanScheduler::prepare_current_dictionary_filters( @@ -1759,17 +1839,37 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( OwnedExpressionConjuncts residual_conjuncts; { SCOPED_TIMER(_scan_profile.dict_filter_build_time); - DictionaryEntryFilterKernel filter_kernel = DictionaryEntryFilterKernel::GENERIC; - RETURN_IF_ERROR(build_dictionary_entry_filter(block_position, *column_schema, - conjunct_it->second, *dictionary_values, - &dictionary_filter, &filter_kernel)); - if (filter_kernel == DictionaryEntryFilterKernel::TYPED_FIXED_WIDTH) { - update_counter_if_not_null(_scan_profile.dict_filter_typed_compare_columns, 1); - } else if (filter_kernel == DictionaryEntryFilterKernel::TYPED_STRING) { - update_counter_if_not_null(_scan_profile.dict_filter_string_compare_columns, 1); - } else if (filter_kernel == DictionaryEntryFilterKernel::VECTORIZED_RUNTIME_FILTER) { - update_counter_if_not_null( - _scan_profile.dict_filter_vectorized_runtime_filter_columns, 1); + const auto cache_key = dictionary_filter_cache_key( + block_position, *column_schema, conjunct_it->second, *dictionary_values); + std::vector cached_filter; + const bool cache_hit = + cache_key.has_value() && _query_dictionary_filter_cache != nullptr && + _query_dictionary_filter_cache->lookup(*cache_key, &cached_filter); + if (cache_hit) { + dictionary_filter.assign(cached_filter.begin(), cached_filter.end()); + update_counter_if_not_null(_scan_profile.query_dict_filter_cache_hits, 1); + } else { + if (cache_key.has_value() && _query_dictionary_filter_cache != nullptr) { + update_counter_if_not_null(_scan_profile.query_dict_filter_cache_misses, 1); + } + DictionaryEntryFilterKernel filter_kernel = DictionaryEntryFilterKernel::GENERIC; + RETURN_IF_ERROR(build_dictionary_entry_filter( + block_position, *column_schema, conjunct_it->second, *dictionary_values, + &dictionary_filter, &filter_kernel)); + if (filter_kernel == DictionaryEntryFilterKernel::TYPED_FIXED_WIDTH) { + update_counter_if_not_null(_scan_profile.dict_filter_typed_compare_columns, 1); + } else if (filter_kernel == DictionaryEntryFilterKernel::TYPED_STRING) { + update_counter_if_not_null(_scan_profile.dict_filter_string_compare_columns, 1); + } else if (filter_kernel == + DictionaryEntryFilterKernel::VECTORIZED_RUNTIME_FILTER) { + update_counter_if_not_null( + _scan_profile.dict_filter_vectorized_runtime_filter_columns, 1); + } + if (cache_key.has_value() && _query_dictionary_filter_cache != nullptr) { + _query_dictionary_filter_cache->insert( + *cache_key, std::vector(dictionary_filter.begin(), + dictionary_filter.end())); + } } residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); } @@ -2706,11 +2806,12 @@ void ParquetScanScheduler::mark_condition_cache_granules(const SelectionVector& Status ParquetScanScheduler::read_next_batch( ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { + const std::vector>& file_schema, Block* file_block, + size_t* rows, bool* eof) { + DORIS_CHECK(_active_request != nullptr); *rows = 0; if (!_pending_predicate_selection.empty()) { - RETURN_IF_ERROR(materialize_pending_predicate_batch(request, file_block, rows)); + RETURN_IF_ERROR(materialize_pending_predicate_batch(*_active_request, file_block, rows)); *eof = false; return Status::OK(); } @@ -2731,9 +2832,10 @@ Status ParquetScanScheduler::read_next_batch( }; while (true) { if (!_has_current_row_group) { + activate_pending_scan_request_at_row_group_boundary(); bool has_row_group = false; - RETURN_IF_ERROR( - open_next_row_group(file_context, file_schema, request, &has_row_group)); + RETURN_IF_ERROR(open_next_row_group(file_context, file_schema, *_active_request, + &has_row_group)); if (!has_row_group) { *eof = true; return Status::OK(); @@ -2769,8 +2871,9 @@ Status ParquetScanScheduler::read_next_batch( const int64_t physical_rows_read = batch_rows; const int64_t batch_first_file_row = _current_row_group_first_row + _current_row_group_rows_read; - RETURN_IF_ERROR(read_current_row_group_batch(file_context, file_schema, batch_rows, request, - batch_first_file_row, file_block, rows)); + RETURN_IF_ERROR(read_current_row_group_batch(file_context, file_schema, batch_rows, + *_active_request, batch_first_file_row, + file_block, rows)); _current_row_group_rows_read += physical_rows_read; _current_range_rows_read += physical_rows_read; if (_current_range_rows_read >= current_range.length) { diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index f0e202d99422a8..19cf0daae4a961 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -42,6 +42,7 @@ class time_zone; namespace doris { class Block; +class QueryDictionaryFilterCache; class RuntimeState; namespace format { @@ -187,7 +188,9 @@ class ParquetScanScheduler { void set_enable_strict_mode(bool enable_strict_mode) { _enable_strict_mode = enable_strict_mode; } - void set_runtime_state(RuntimeState* runtime_state) { _runtime_state = runtime_state; } + void set_runtime_state(RuntimeState* runtime_state); + void set_scan_request(std::shared_ptr request); + void queue_scan_request(std::shared_ptr request); // Release row-group readers before the owning RuntimeProfile is reported. Native readers // publish their accumulated page/decode statistics from their destructor. void close() { reset_current_row_group(); } @@ -204,13 +207,13 @@ class ParquetScanScheduler { Status read_next_batch(ParquetFileContext& file_context, const std::vector>& file_schema, - const format::FileScanRequest& request, Block* file_block, size_t* rows, - bool* eof); + Block* file_block, size_t* rows, bool* eof); private: static constexpr size_t PROFILE_FLUSH_BATCH_INTERVAL = 16; void reset_current_row_group(); + void activate_pending_scan_request_at_row_group_boundary(); void flush_current_reader_profiles(); bool finish_current_reader_batch_profiles(); const detail::PredicateConjunctSchedule& predicate_conjunct_schedule( @@ -304,12 +307,15 @@ class ParquetScanScheduler { bool _enable_strict_mode = false; bool _enable_bloom_filter = false; RuntimeState* _runtime_state = nullptr; + QueryDictionaryFilterCache* _query_dictionary_filter_cache = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without // reallocating selection indices, dense filter bytes, or compacted-column positions. SelectionVector _selection; std::vector _read_column_positions_scratch; const format::FileScanRequest* _predicate_schedule_request = nullptr; + std::shared_ptr _active_request; + std::shared_ptr _pending_request; detail::PredicateConjunctSchedule _predicate_schedule; std::vector _predicate_positions_scratch; std::unordered_map _predicate_indices_by_position_scratch; diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 60eb0dc3d92404..6b686597fa600e 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -738,7 +738,6 @@ Status NativeColumnReader::select_with_fixed_width_filter( DORIS_CHECK(used_filter != nullptr); DORIS_CHECK(execution_kind != nullptr); RETURN_IF_ERROR(validate_selected_span(batch_rows)); - RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); const uint8_t* filter_data = nullptr; RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); int64_t rows_read = 0; @@ -768,7 +767,6 @@ Status NativeColumnReader::select_with_runtime_filter( DORIS_CHECK(row_filter != nullptr); DORIS_CHECK(used_filter != nullptr); RETURN_IF_ERROR(validate_selected_span(batch_rows)); - RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); row_filter->clear(); *used_filter = false; if (_nested || conjuncts.empty() || !std::ranges::all_of(conjuncts, [&](const auto& conjunct) { diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 033478875fad95..8ae53f5fcf9f6f 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -68,17 +68,18 @@ class SelectionVector { _data = data; _size = count; _identity = data == nullptr; + _mutable_data_exposed = data != nullptr; ++_generation; } void resize(size_t count) { - _owned.resize(count); - _data = _owned.data(); + // Identity is the overwhelmingly common initial state. Keep it implicit until a caller + // actually changes an index, avoiding one write per source row for every scanner batch. + _owned.clear(); + _data = nullptr; _size = count; - for (size_t idx = 0; idx < count; ++idx) { - _data[idx] = static_cast(idx); - } _identity = true; + _mutable_data_exposed = false; ++_generation; } @@ -87,6 +88,7 @@ class SelectionVector { _data = nullptr; _size = 0; _identity = true; + _mutable_data_exposed = false; ++_generation; } @@ -95,9 +97,11 @@ class SelectionVector { bool is_set() const { return _data != nullptr; } Index* data() { + _materialize_identity(); // A mutable pointer can change indices without set_index(), so identity can no longer be // proven until resize() rebuilds it. This keeps the O(1) dense fast path conservative. _identity = false; + _mutable_data_exposed = true; ++_generation; return _data; } @@ -112,6 +116,7 @@ class SelectionVector { } void set_index(size_t idx, Index value) { + _materialize_identity(); _data[idx] = value; if (value != idx) { _identity = false; @@ -119,10 +124,18 @@ class SelectionVector { ++_generation; } + size_t compact_with_row_filter(const uint8_t* filter, size_t count) { + return _compact(filter, count, true); + } + + size_t compact_with_selection_filter(const uint8_t* filter, size_t count) { + return _compact(filter, count, false); + } + Status materialize_filter(size_t count, int64_t batch_rows, const uint8_t** filter) const { DORIS_CHECK(filter != nullptr); if (batch_rows >= 0 && std::cmp_equal(count, batch_rows) && _identity && - (_data == nullptr || count <= _size)) { + (_size == 0 || count <= _size)) { // A proven identity selection is equivalent to no FilterMap. Returning nullptr avoids // constructing and rescanning one dense byte per source row. *filter = nullptr; @@ -157,6 +170,17 @@ class SelectionVector { return Status::InvalidArgument("Parquet selection count {} exceeds vector size {}", count, _size); } + if (_data == nullptr && _size != 0 && count > _size) { + return Status::InvalidArgument("Parquet selection count {} exceeds vector size {}", + count, _size); + } + if (_identity) { + return Status::OK(); + } + if (!_mutable_data_exposed && _verified_generation == _generation && + _verified_count == count && _verified_batch_rows == batch_rows) { + return Status::OK(); + } size_t previous = 0; for (size_t idx = 0; idx < count; ++idx) { const size_t current = get_index(idx); @@ -173,19 +197,65 @@ class SelectionVector { } previous = current; } + if (!_mutable_data_exposed) { + _verified_generation = _generation; + _verified_count = count; + _verified_batch_rows = batch_rows; + } return Status::OK(); } private: + void _materialize_identity() { + if (_data != nullptr) { + return; + } + _owned.resize(_size); + _data = _owned.data(); + for (size_t idx = 0; idx < _size; ++idx) { + _data[idx] = static_cast(idx); + } + } + + size_t _compact(const uint8_t* filter, size_t count, bool filter_uses_row_index) { + DORIS_CHECK(filter != nullptr); + DORIS_CHECK(count <= _size); + Index* source = _data; + if (_data == nullptr) { + _owned.resize(_size); + _data = _owned.data(); + } + size_t output = 0; + bool remains_identity = true; + for (size_t position = 0; position < count; ++position) { + const Index row = source == nullptr ? static_cast(position) : source[position]; + const size_t filter_position = filter_uses_row_index ? row : position; + if (filter[filter_position] != 0) { + _data[output] = row; + remains_identity &= row == output; + ++output; + } + } + // Compaction is one logical mutation. Invalidating caches once is important when several + // predicates successively refine a wide batch. + _identity = remains_identity; + ++_generation; + return output; + } + std::vector _owned; Index* _data = nullptr; size_t _size = 0; bool _identity = true; + bool _mutable_data_exposed = false; uint64_t _generation = 0; mutable std::vector _filter; mutable uint64_t _filter_generation = std::numeric_limits::max(); mutable size_t _filter_count = 0; mutable int64_t _filter_batch_rows = -1; + mutable uint64_t _verified_generation = std::numeric_limits::max(); + mutable size_t _verified_count = 0; + mutable int64_t _verified_batch_rows = -1; }; inline void selection_to_ranges(const SelectionVector& selection, uint16_t selected_rows, diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 8a5da295fddd4a..1768b617736759 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -765,6 +765,50 @@ Status TableReader::_build_table_filters_from_conjuncts() { return Status::OK(); } +Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { + _conjuncts = std::move(conjuncts); + if (_data_reader.reader == nullptr) { + // The split is prepared but its physical reader has not opened yet. open_reader() will use + // this newest snapshot directly, so no pending request is needed. + return Status::OK(); + } + if (!_data_reader.reader->supports_scan_request_refresh()) { + return Status::OK(); + } + + RETURN_IF_ERROR(_build_table_filters_from_conjuncts()); + auto refreshed_request = std::make_shared(); + RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( + _table_filters, _projected_columns, refreshed_request.get(), _runtime_state, + _file_scan_request == nullptr ? nullptr : &_file_scan_request->local_positions)); + if (_push_down_agg_type == TPushAggOp::type::COUNT && _push_down_count_columns.has_value() && + _push_down_count_columns->empty()) { + for (const auto& column : refreshed_request->non_predicate_columns) { + refreshed_request->count_star_placeholder_columns.push_back(column.column_id()); + } + } + RETURN_IF_ERROR(customize_file_scan_request(refreshed_request.get())); + RETURN_IF_ERROR(_open_local_filter_exprs(*refreshed_request)); + if (_file_scan_request == nullptr || + refreshed_request->local_positions != _file_scan_request->local_positions) { + // A reader cannot reinterpret columns already materialized with another block layout. + // Keep scanner-level filtering as the correctness fallback for this uncommon hidden-slot + // shape instead of switching an incompatible request mid-file. + return Status::OK(); + } + + if (_condition_cache_ctx != nullptr && !_condition_cache_ctx->is_hit) { + // Rows before and after a late RF were evaluated by different predicate snapshots. Such a + // partial MISS bitmap must never be published under either snapshot's cache key. + _condition_cache = nullptr; + _condition_cache_ctx = nullptr; + _data_reader.reader->set_condition_cache_context(nullptr); + } + RETURN_IF_ERROR(_data_reader.reader->queue_scan_request(refreshed_request)); + _file_scan_request = std::move(refreshed_request); + return Status::OK(); +} + Status TableReader::_open_local_filter_exprs(const FileScanRequest& file_request) { RowDescriptor row_desc; for (const auto& conjunct : file_request.conjuncts) { diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 6d679ec2d797cf..97f8d0d54d2729 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -223,6 +223,10 @@ class TableReader { // 2. Parse delete predicates from split/task information, which will be used for later dynamic filtering and delete handling. virtual Status prepare_split(const SplitReadOptions& options); + // Refresh row-level predicates for an already prepared split. Physical readers that support + // this operation decide the safe boundary at which the new immutable request becomes active. + Status refresh_conjuncts(VExprContextSPtrs conjuncts); + virtual bool current_split_pruned() const { return _current_split_pruned; } virtual bool current_split_uses_metadata_count() const { return _current_split_uses_metadata_count; @@ -465,6 +469,7 @@ class TableReader { RETURN_IF_ERROR(_open_local_filter_exprs(*file_request)); _data_reader.file_block_layout.clear(); _data_reader.block_template.clear(); + _file_scan_request.reset(); _data_reader.file_block_layout.resize(file_request->local_positions.size()); // 4. Build file block layout from file schema and column mapping. The layout describes @@ -517,7 +522,8 @@ class TableReader { SCOPED_TIMER(_profile.file_reader_open_timer); RETURN_IF_ERROR(_data_reader.reader->open(file_request)); } - RETURN_IF_ERROR(_init_reader_condition_cache(*file_request)); + _file_scan_request = std::move(file_request); + RETURN_IF_ERROR(_init_reader_condition_cache(*_file_scan_request)); return Status::OK(); } @@ -763,6 +769,7 @@ class TableReader { _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); _data_reader.block_template.clear(); + _file_scan_request.reset(); _current_task.reset(); _current_file_description.reset(); _current_reader_reached_eof = false; @@ -1877,6 +1884,9 @@ class TableReader { Block block_template; }; DataReader _data_reader; + // Latest immutable request queued to the physical reader. The file-block layout remains fixed + // for the split even while predicates are refreshed at a reader-defined granule boundary. + std::shared_ptr _file_scan_request; std::vector _projected_columns; std::unique_ptr _current_task; std::optional _current_file_description; diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp index 29748d7639d06a..e36310fa4879a4 100644 --- a/be/src/runtime/query_context.cpp +++ b/be/src/runtime/query_context.cpp @@ -40,6 +40,7 @@ #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/memory/heap_profiler.h" +#include "runtime/query_dictionary_filter_cache.h" #include "runtime/runtime_query_statistics_mgr.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" @@ -261,6 +262,13 @@ QueryContext::~QueryContext() { elapsed_ms, mem_tracker_msg); } +QueryDictionaryFilterCache& QueryContext::query_dictionary_filter_cache() { + std::call_once(_query_dictionary_filter_cache_once, [this] { + _query_dictionary_filter_cache = std::make_unique(); + }); + return *_query_dictionary_filter_cache; +} + void QueryContext::set_ready_to_execute(Status reason) { set_execution_dependency_ready(); _exec_status.update(reason); diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h index 3ef113c1fd1097..3964c28c84dfda 100644 --- a/be/src/runtime/query_context.h +++ b/be/src/runtime/query_context.h @@ -50,6 +50,7 @@ class PipelineTask; class QueryTaskController; class Dependency; class RecCTEScanLocalState; +class QueryDictionaryFilterCache; struct ReportStatusRequest { const Status status; @@ -152,6 +153,8 @@ class QueryContext : public std::enable_shared_from_this { return _runtime_predicates.find(source_node_id)->second; } + QueryDictionaryFilterCache& query_dictionary_filter_cache(); + void init_runtime_predicates(const std::vector& topn_filter_descs) { for (auto desc : topn_filter_descs) { _runtime_predicates.try_emplace(desc.source_node_id, desc); @@ -325,6 +328,8 @@ class QueryContext : public std::enable_shared_from_this { void _init_query_mem_tracker(); std::unordered_map _runtime_predicates; + std::once_flag _query_dictionary_filter_cache_once; + std::unique_ptr _query_dictionary_filter_cache; std::unique_ptr _runtime_filter_mgr; const TQueryOptions _query_options; diff --git a/be/src/runtime/query_dictionary_filter_cache.h b/be/src/runtime/query_dictionary_filter_cache.h new file mode 100644 index 00000000000000..83203b64e3f5f8 --- /dev/null +++ b/be/src/runtime/query_dictionary_filter_cache.h @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "core/data_type/define_primitive_type.h" + +namespace doris { + +struct QueryDictionaryFilterCacheKey { + uint64_t expression_digest = 0; + uint64_t dictionary_hash_low = 0; + uint64_t dictionary_hash_high = 0; + uint32_t dictionary_entries = 0; + PrimitiveType primitive_type = INVALID_TYPE; + + bool operator==(const QueryDictionaryFilterCacheKey&) const = default; +}; + +struct QueryDictionaryFilterCacheKeyHash { + size_t operator()(const QueryDictionaryFilterCacheKey& key) const { + size_t hash = key.expression_digest; + hash ^= key.dictionary_hash_low + 0x9e3779b97f4a7c15ULL + (hash << 6) + (hash >> 2); + hash ^= key.dictionary_hash_high + 0x9e3779b97f4a7c15ULL + (hash << 6) + (hash >> 2); + hash ^= static_cast(key.dictionary_entries) << 1; + hash ^= static_cast(key.primitive_type) << 17; + return hash; + } +}; + +// Query-scoped dictionaries often repeat across row groups and files written by one job. Cache the +// predicate result by dictionary content rather than by file offsets, which are not stable aliases. +class QueryDictionaryFilterCache { +public: + static constexpr size_t DEFAULT_MAX_BYTES = 16 * 1024 * 1024; + + explicit QueryDictionaryFilterCache(size_t max_bytes = DEFAULT_MAX_BYTES) + : _max_bytes(max_bytes) {} + + bool lookup(const QueryDictionaryFilterCacheKey& key, std::vector* result) const { + std::lock_guard lock(_mutex); + const auto it = _entries.find(key); + if (it == _entries.end()) { + return false; + } + *result = it->second; + return true; + } + + bool insert(const QueryDictionaryFilterCacheKey& key, std::vector result) { + const size_t charge = _entry_charge(result.size()); + if (result.empty() || charge > _max_bytes) { + return false; + } + std::lock_guard lock(_mutex); + if (_entries.contains(key)) { + return true; + } + if (charge > _max_bytes - _memory_charge) { + return false; + } + _memory_charge += charge; + _entries.emplace(key, std::move(result)); + return true; + } + +private: + static constexpr size_t _entry_charge(size_t bitmap_bytes) { + // Include fixed entry ownership so many tiny dictionaries cannot bypass the query cap. + return bitmap_bytes + sizeof(QueryDictionaryFilterCacheKey) + sizeof(std::vector) + + 2 * sizeof(void*); + } + + const size_t _max_bytes; + mutable std::mutex _mutex; + size_t _memory_charge = 0; + std::unordered_map, + QueryDictionaryFilterCacheKeyHash> + _entries; +}; + +} // namespace doris diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 9de951ec5e47dd..0e44569c6a80db 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -33,6 +33,7 @@ #include "format_v2/parquet/reader/global_rowid_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" #include "format_v2/parquet/selection_vector.h" +#include "runtime/query_dictionary_filter_cache.h" #include "storage/utils.h" namespace doris::format::parquet { @@ -179,11 +180,45 @@ TEST(SelectionVectorTest, MaterializedFilterIsReusedUntilSelectionChanges) { TEST(SelectionVectorTest, IdentitySelectionDoesNotMaterializeFilter) { SelectionVector selection(4); + EXPECT_FALSE(selection.is_set()); const uint8_t* filter = reinterpret_cast(1); ASSERT_TRUE(selection.materialize_filter(4, 4, &filter).ok()); EXPECT_EQ(filter, nullptr); } +TEST(SelectionVectorTest, BulkCompactionSupportsBothFilterCoordinates) { + SelectionVector selection(6); + const uint8_t row_filter[] = {0, 1, 1, 0, 1, 0}; + ASSERT_EQ(selection.compact_with_row_filter(row_filter, 6), 3); + EXPECT_EQ(selection.get_index(0), 1); + EXPECT_EQ(selection.get_index(1), 2); + EXPECT_EQ(selection.get_index(2), 4); + + const uint8_t compact_filter[] = {1, 0, 1}; + ASSERT_EQ(selection.compact_with_selection_filter(compact_filter, 3), 2); + EXPECT_EQ(selection.get_index(0), 1); + EXPECT_EQ(selection.get_index(1), 4); + EXPECT_TRUE(selection.verify(2, 6).ok()); +} + +TEST(QueryDictionaryFilterCacheTest, ReusesBitmapWithinMemoryLimit) { + QueryDictionaryFilterCache cache(128); + QueryDictionaryFilterCacheKey key {.expression_digest = 11, + .dictionary_hash_low = 22, + .dictionary_hash_high = 33, + .dictionary_entries = 4, + .primitive_type = TYPE_INT}; + std::vector result; + EXPECT_FALSE(cache.lookup(key, &result)); + EXPECT_TRUE(cache.insert(key, {1, 0, 1, 0})); + ASSERT_TRUE(cache.lookup(key, &result)); + EXPECT_EQ(result, std::vector({1, 0, 1, 0})); + + QueryDictionaryFilterCache tiny_cache(2); + EXPECT_FALSE(tiny_cache.insert(key, {1, 0, 1, 0})); + EXPECT_FALSE(tiny_cache.lookup(key, &result)); +} + TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { CursorColumnReader reader; SelectionVector selection(3); @@ -243,6 +278,23 @@ TEST(ParquetColumnReaderControlTest, SchedulerOrsPageCrossingOncePerBatch) { EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); } +TEST(ParquetColumnReaderControlTest, PendingRequestActivatesOnlyAtRowGroupBoundary) { + ParquetScanScheduler scheduler; + auto initial = std::make_shared(); + auto refreshed = std::make_shared(); + refreshed->predicate_only_columns.push_back(format::LocalColumnId(7)); + + scheduler.set_scan_request(initial); + scheduler._has_current_row_group = true; + scheduler.queue_scan_request(refreshed); + scheduler.activate_pending_scan_request_at_row_group_boundary(); + EXPECT_EQ(scheduler._active_request, initial); + + scheduler._has_current_row_group = false; + scheduler.activate_pending_scan_request_at_row_group_boundary(); + EXPECT_EQ(scheduler._active_request, refreshed); +} + TEST(ParquetColumnReaderControlTest, PendingOutputDrainsBeforePageCrossingSample) { ParquetScanScheduler scheduler; scheduler._batch_size = 1; diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 75733164ed268f..a71a6a6573ae98 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -3193,6 +3193,8 @@ TEST_F(ParquetScanTest, PredicateOnlyDictionaryTopNUsesDictionaryIds) { EXPECT_EQ(counter_value(profile, "DictFilterColumns"), 1); EXPECT_EQ(counter_value(profile, "RowsFilteredByDictFilter"), 3); EXPECT_EQ(counter_value(profile, "DictionaryPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "QueryDictionaryFilterCacheHits"), 0); + EXPECT_EQ(counter_value(profile, "QueryDictionaryFilterCacheMisses"), 0); prepared.conjunct->close(); } diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index bb0db9329a2a1c..9615d82b0c36cf 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1050,6 +1050,7 @@ struct FakeFileReaderState { int init_count = 0; int open_count = 0; int close_count = 0; + int refresh_count = 0; int64_t total_rows = 2; int64_t aggregate_count = -1; int64_t condition_cache_base_granule = 0; @@ -1060,6 +1061,7 @@ struct FakeFileReaderState { bool stop_during_read = false; bool not_found_during_init = false; std::shared_ptr last_request; + std::shared_ptr pending_request; std::optional last_aggregate_request; std::shared_ptr condition_cache_ctx; std::shared_ptr io_ctx; @@ -1101,6 +1103,14 @@ class FakeFileReader final : public FileReader { return Status::OK(); } + bool supports_scan_request_refresh() const override { return true; } + + Status queue_scan_request(std::shared_ptr request) override { + _state->pending_request = std::move(request); + ++_state->refresh_count; + return Status::OK(); + } + Status get_block(Block* file_block, size_t* rows, bool* eof) override { DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); @@ -1610,6 +1620,55 @@ TEST(TableReaderTest, PrepareSplitReplacesInitialConjunctSnapshot) { ASSERT_TRUE(reader.close().ok()); } +TEST(TableReaderTest, ActiveReaderQueuesRefreshedRuntimeFilterRequest) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + file_schema.push_back(make_file_column(1, "value", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + projected_columns.push_back(make_table_column(1, "value", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->eof_with_first_batch = false; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + VExprContextSPtrs refreshed {VExprContext::create_shared( + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 1)))}; + ASSERT_TRUE(reader.refresh_conjuncts(std::move(refreshed)).ok()); + ASSERT_EQ(fake_state->refresh_count, 1); + ASSERT_NE(fake_state->pending_request, nullptr); + EXPECT_EQ(fake_state->pending_request->local_positions, + fake_state->last_request->local_positions); + EXPECT_EQ(projection_ids(fake_state->pending_request->predicate_columns), + std::vector({0})); + EXPECT_EQ(projection_ids(fake_state->pending_request->non_predicate_columns), + std::vector({1})); + ASSERT_EQ(fake_state->pending_request->conjuncts.size(), 1); + EXPECT_TRUE(fake_state->pending_request->conjuncts.front()->root()->is_rf_wrapper()); + ASSERT_TRUE(reader.close().ok()); +} + TEST(TableReaderTest, RefreshedConjunctDisablesTableLevelCount) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); From c6585fa01714cc25472de7b2c30215c693a66eb9 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 30 Jul 2026 19:18:01 +0800 Subject: [PATCH 02/10] [fix](scan) initialize refreshed JNI predicates --- be/src/format_v2/jni/jni_table_reader.cpp | 13 ++++++++++ be/src/format_v2/jni/jni_table_reader.h | 1 + be/src/format_v2/table_reader.h | 2 +- .../format_v2/jni/jni_table_reader_test.cpp | 26 +++++++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index 1c691fc3129c95..f287e8910bc70d 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -77,6 +77,19 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) { return _open_jni_scanner(); } +Status JniTableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { + if (_scanner_opened) { + RowDescriptor row_desc; + for (const auto& conjunct : conjuncts) { + // JNI readers bypass TableReader::open_reader(), so a late predicate would otherwise + // replace the active snapshot without initializing its executable function state. + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, row_desc)); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + } + } + return TableReader::refresh_conjuncts(std::move(conjuncts)); +} + Status JniTableReader::get_block(Block* output_block, bool* eos) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index 5e48270515f996..34124d46245364 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -48,6 +48,7 @@ class JniTableReader : public TableReader { Status init(TableReadOptions&& options) override; Status prepare_split(const SplitReadOptions& options) override; + Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; Status get_block(Block* block, bool* eos) override; Status abort_split() override; Status close() override; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 97f8d0d54d2729..ad081eba9f3499 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -225,7 +225,7 @@ class TableReader { // Refresh row-level predicates for an already prepared split. Physical readers that support // this operation decide the safe boundary at which the new immutable request becomes active. - Status refresh_conjuncts(VExprContextSPtrs conjuncts); + virtual Status refresh_conjuncts(VExprContextSPtrs conjuncts); virtual bool current_split_pruned() const { return _current_split_pruned; } virtual bool current_split_uses_metadata_count() const { diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp index 95eda4a557e790..5d10912591d747 100644 --- a/be/test/format_v2/jni/jni_table_reader_test.cpp +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -27,6 +27,9 @@ #include #include +#include "core/data_type/data_type_number.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" #include "io/io_common.h" namespace doris::format { @@ -201,6 +204,29 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { EXPECT_TRUE(reader.TEST_scanner_opened()); } +TEST(JniTableReaderTest, RefreshedConjunctIsReadyBeforeFilteringOpenScanner) { + FakeJniTableReader reader; + ASSERT_TRUE(init_reader(&reader, nullptr).ok()); + ASSERT_TRUE(reader.prepare_split({ + .partition_values = {}, + .conjuncts = std::nullopt, + .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, + .condition_cache_digest = std::nullopt, + .cache = nullptr, + .current_range = {}, + .current_split_format = FileFormat::JNI, + .global_rowid_context = std::nullopt, + }) + .ok()); + + auto refreshed = VExprContext::create_shared( + VSlotRef::create_shared(0, 0, 0, std::make_shared(), "filter_column")); + ASSERT_FALSE(refreshed->root()->ready_status().ok()); + ASSERT_TRUE(reader.refresh_conjuncts({refreshed}).ok()); + EXPECT_TRUE(refreshed->root()->ready_status().ok()); +} + TEST(JniTableReaderTest, CommonLifecycleTimersContainJniLifecycleWork) { constexpr auto delay = std::chrono::milliseconds(8); RuntimeProfile profile("JniLifecycleContainment"); From 3eb01f37c4463071a070f90dec6da13262d661ca Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 30 Jul 2026 20:21:40 +0800 Subject: [PATCH 03/10] [fix](scan) preserve late predicate reader state --- be/src/format_v2/table/hudi_reader.cpp | 12 +++ be/src/format_v2/table/hudi_reader.h | 1 + be/src/format_v2/table/paimon_reader.cpp | 12 +++ be/src/format_v2/table/paimon_reader.h | 1 + be/src/format_v2/table_reader.cpp | 64 ++++++++++++- .../format_v2/parquet/parquet_scan_test.cpp | 95 ++++++++++++++++++- be/test/format_v2/table/hudi_reader_test.cpp | 45 +++++++++ .../format_v2/table/paimon_reader_test.cpp | 44 +++++++++ be/test/format_v2/table_reader_test.cpp | 49 ++++++++++ 9 files changed, 315 insertions(+), 8 deletions(-) diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index ee24d0f9ad7d02..838d98b5e52e14 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -80,6 +80,18 @@ Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) return _current_split_reader->prepare_split(options); } +Status HudiHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { + RETURN_IF_ERROR(format::TableReader::refresh_conjuncts(std::move(conjuncts))); + if (_current_split_reader == nullptr) { + return Status::OK(); + } + VExprContextSPtrs child_conjuncts; + RETURN_IF_ERROR(_clone_conjuncts(&child_conjuncts)); + // The hybrid wrapper owns no physical reader; forward a clone so the active child, rather than + // only the wrapper snapshot, observes late predicates for the remainder of this split. + return _current_split_reader->refresh_conjuncts(std::move(child_conjuncts)); +} + Status HudiHybridReader::get_block(Block* block, bool* eos) { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->get_block(block, eos); diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index dbb6f5e8231043..c06e1b238b62c4 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -59,6 +59,7 @@ class HudiHybridReader final : public format::TableReader { Status init(format::TableReadOptions&& options) override; Status prepare_split(const format::SplitReadOptions& options) override; + Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; bool current_split_uses_metadata_count() const override; diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index a3f4092a470263..93183af178babb 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -115,6 +115,18 @@ Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options return _current_split_reader->prepare_split(options); } +Status PaimonHybridReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { + RETURN_IF_ERROR(format::TableReader::refresh_conjuncts(std::move(conjuncts))); + if (_current_split_reader == nullptr) { + return Status::OK(); + } + VExprContextSPtrs child_conjuncts; + RETURN_IF_ERROR(_clone_conjuncts(&child_conjuncts)); + // The hybrid wrapper owns no physical reader; forward a clone so the active child, rather than + // only the wrapper snapshot, observes late predicates for the remainder of this split. + return _current_split_reader->refresh_conjuncts(std::move(child_conjuncts)); +} + Status PaimonHybridReader::get_block(Block* block, bool* eos) { DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->get_block(block, eos); diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index 823fa6540d280c..8570f2efba624e 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -65,6 +65,7 @@ class PaimonHybridReader final : public format::TableReader { Status init(format::TableReadOptions&& options) override; Status prepare_split(const format::SplitReadOptions& options) override; + Status refresh_conjuncts(VExprContextSPtrs conjuncts) override; Status get_block(Block* block, bool* eos) override; bool current_split_pruned() const override; bool current_split_uses_metadata_count() const override; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 1768b617736759..c542cb908f53d7 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -765,6 +765,53 @@ Status TableReader::_build_table_filters_from_conjuncts() { return Status::OK(); } +namespace { + +bool same_scan_projection(const LocalColumnIndex& lhs, const LocalColumnIndex& rhs) { + if (lhs.index != rhs.index || lhs.project_all_children != rhs.project_all_children || + lhs.children.size() != rhs.children.size()) { + return false; + } + for (size_t index = 0; index < lhs.children.size(); ++index) { + if (!same_scan_projection(lhs.children[index], rhs.children[index])) { + return false; + } + } + return true; +} + +const LocalColumnIndex* find_scan_projection(const FileScanRequest& request, + LocalColumnId column_id) { + const auto find_by_id = [column_id](const std::vector& projections) { + return std::ranges::find_if(projections, [column_id](const LocalColumnIndex& projection) { + return projection.column_id() == column_id; + }); + }; + auto it = find_by_id(request.predicate_columns); + if (it != request.predicate_columns.end()) { + return &*it; + } + it = find_by_id(request.non_predicate_columns); + return it == request.non_predicate_columns.end() ? nullptr : &*it; +} + +bool same_physical_scan_layout(const FileScanRequest& lhs, const FileScanRequest& rhs) { + if (lhs.local_positions != rhs.local_positions) { + return false; + } + for (const auto& [column_id, _] : lhs.local_positions) { + const auto* lhs_projection = find_scan_projection(lhs, column_id); + const auto* rhs_projection = find_scan_projection(rhs, column_id); + if (lhs_projection == nullptr || rhs_projection == nullptr || + !same_scan_projection(*lhs_projection, *rhs_projection)) { + return false; + } + } + return true; +} + +} // namespace + Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { _conjuncts = std::move(conjuncts); if (_data_reader.reader == nullptr) { @@ -777,8 +824,15 @@ Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { } RETURN_IF_ERROR(_build_table_filters_from_conjuncts()); + // create_scan_request() rebuilds mapping projections in place. Build late predicates with an + // isolated mapper so the active row group cannot observe an unprepared or incompatible mapper + // before its physical request reaches the reader's safe activation boundary. + auto refreshed_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + DORIS_CHECK(refreshed_mapper != nullptr); + RETURN_IF_ERROR(refreshed_mapper->create_mapping(_projected_columns, _partition_values, + _data_reader.file_schema)); auto refreshed_request = std::make_shared(); - RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( + RETURN_IF_ERROR(refreshed_mapper->create_scan_request( _table_filters, _projected_columns, refreshed_request.get(), _runtime_state, _file_scan_request == nullptr ? nullptr : &_file_scan_request->local_positions)); if (_push_down_agg_type == TPushAggOp::type::COUNT && _push_down_count_columns.has_value() && @@ -788,14 +842,14 @@ Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { } } RETURN_IF_ERROR(customize_file_scan_request(refreshed_request.get())); - RETURN_IF_ERROR(_open_local_filter_exprs(*refreshed_request)); if (_file_scan_request == nullptr || - refreshed_request->local_positions != _file_scan_request->local_positions) { + !same_physical_scan_layout(*refreshed_request, *_file_scan_request)) { // A reader cannot reinterpret columns already materialized with another block layout. - // Keep scanner-level filtering as the correctness fallback for this uncommon hidden-slot - // shape instead of switching an incompatible request mid-file. + // Keep scanner-level filtering as the correctness fallback for hidden slots or nested + // projections instead of switching an incompatible physical shape mid-file. return Status::OK(); } + RETURN_IF_ERROR(_open_local_filter_exprs(*refreshed_request)); if (_condition_cache_ctx != nullptr && !_condition_cache_ctx->is_hit) { // Rows before and after a late RF were evaluated by different predicate snapshots. Such a diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index a71a6a6573ae98..fccad99fd50936 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -1398,6 +1398,22 @@ void write_dictionary_int_pair_parquet_file(const std::string& file_path) { write_table(file_path, table, 6, true, false, false); } +void write_dictionary_int_pair_parquet_file(const std::string& file_path, + const std::vector& dictionary_values) { + std::vector scores(dictionary_values.size()); + for (size_t index = 0; index < scores.size(); ++index) { + scores[index] = static_cast((index + 1) * 10); + } + auto schema = arrow::schema({ + arrow::field("id", arrow::int32(), false), + arrow::field("score", arrow::int32(), false), + }); + auto table = arrow::Table::Make( + schema, {build_int32_array(dictionary_values), build_int32_array(scores)}); + write_table(file_path, table, static_cast(dictionary_values.size()), true, false, + false); +} + void write_dictionary_bigint_pair_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int64(), false), @@ -3163,17 +3179,90 @@ TEST_F(ParquetScanTest, PredicateOnlyDictionaryRangeSkipsTypedValueMaterializati conjunct->close(); } +TEST_F(ParquetScanTest, QueryDictionaryFilterCacheUsesProductionKeysAndBitmaps) { + struct ScanResult { + std::vector scores; + int64_t cache_hits = 0; + int64_t cache_misses = 0; + }; + + auto query_context = MockQueryContext::create(); + auto scan = [&](int32_t lower_bound) { + RuntimeProfile profile("profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + state._query_ctx = query_context.get(); + auto reader = create_reader(0, -1, &profile); + EXPECT_TRUE(reader->init(&state).ok()); + + std::vector schema; + EXPECT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + EXPECT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + EXPECT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + auto conjunct = + create_int32_function_conjunct(0, "gt", TExprOpcode::GT, lower_bound, false); + EXPECT_TRUE(conjunct->prepare(&state, RowDescriptor()).ok()); + EXPECT_TRUE(conjunct->open(&state).ok()); + request->conjuncts.push_back(conjunct); + EXPECT_TRUE(reader->open(request).ok()); + + ScanResult result; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + EXPECT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto& score_column = int32_data_column(*block.get_by_position(1).column); + for (size_t row = 0; row < rows; ++row) { + result.scores.push_back(score_column.get_element(row)); + } + } + result.cache_hits = counter_value(profile, "QueryDictionaryFilterCacheHits"); + result.cache_misses = counter_value(profile, "QueryDictionaryFilterCacheMisses"); + conjunct->close(); + EXPECT_TRUE(reader->close().ok()); + return result; + }; + + write_dictionary_int_pair_parquet_file(_file_path); + const auto first = scan(2); + EXPECT_EQ(first.scores, std::vector({30, 40, 50, 60})); + EXPECT_EQ(first.cache_hits, 0); + EXPECT_EQ(first.cache_misses, 1); + + const auto repeated = scan(2); + EXPECT_EQ(repeated.scores, first.scores); + EXPECT_EQ(repeated.cache_hits, 1); + EXPECT_EQ(repeated.cache_misses, 0); + + const auto changed_predicate = scan(3); + EXPECT_EQ(changed_predicate.scores, std::vector({40, 50, 60})); + EXPECT_EQ(changed_predicate.cache_hits, 0); + EXPECT_EQ(changed_predicate.cache_misses, 1); + + write_dictionary_int_pair_parquet_file(_file_path, {1, 2, 7, 8, 9, 10}); + const auto changed_dictionary = scan(2); + EXPECT_EQ(changed_dictionary.scores, std::vector({30, 40, 50, 60})); + EXPECT_EQ(changed_dictionary.cache_hits, 0); + EXPECT_EQ(changed_dictionary.cache_misses, 1); +} + TEST_F(ParquetScanTest, PredicateOnlyDictionaryTopNUsesDictionaryIds) { write_dictionary_int_pair_parquet_file(_file_path); RuntimeProfile profile("profile"); - auto reader = create_reader(0, -1, &profile); RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto prepared = + create_topn_conjunct(&state, 0, make_nullable(std::make_shared()), + Field::create_field(3)); + ASSERT_NE(state.get_query_ctx(), nullptr); + auto reader = create_reader(0, -1, &profile); ASSERT_TRUE(reader->init(&state).ok()); std::vector schema; ASSERT_TRUE(reader->get_schema(&schema).ok()); - auto prepared = - create_topn_conjunct(&state, 0, schema[0].type, Field::create_field(3)); + ASSERT_TRUE(schema[0].type->equals(*prepared.conjunct->root()->children()[0]->data_type())); auto request = std::make_shared(); format::FileScanRequestBuilder request_builder(request.get()); ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 96126281744f5a..e75eee47be39d6 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -141,6 +141,18 @@ class SlowInitTableReader final : public TableReader { } }; +class RefreshTrackingTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status refresh_conjuncts(VExprContextSPtrs conjuncts) override { + ++refresh_count; + return TableReader::refresh_conjuncts(std::move(conjuncts)); + } + + int refresh_count = 0; +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -269,6 +281,39 @@ TEST(HudiHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { EXPECT_EQ(reader.condition_cache_hit_count(), 9); } +TEST(HudiHybridReaderTest, ForwardsLatePredicatesToActiveChild) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + hudi::HudiHybridReader reader; + RefreshTrackingTableReader* child = nullptr; + reader.TEST_set_child_reader_factories( + [&] { + auto tracking = std::make_unique(); + child = tracking.get(); + return tracking; + }, + [] { return std::make_unique(); }); + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_split_format = FileFormat::PARQUET; + split.current_range.__set_format_type(TFileFormatType::FORMAT_PARQUET); + ASSERT_TRUE(reader.prepare_split(split).ok()); + ASSERT_NE(child, nullptr); + ASSERT_TRUE(reader.refresh_conjuncts({}).ok()); + EXPECT_EQ(child->refresh_count, 1); +} + TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_hudi_hybrid_count_star_test"; diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 6ebec512f42fdf..32b82ab12acbe3 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -79,6 +79,18 @@ class SlowInitTableReader final : public TableReader { } }; +class RefreshTrackingTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status refresh_conjuncts(VExprContextSPtrs conjuncts) override { + ++refresh_count; + return TableReader::refresh_conjuncts(std::move(conjuncts)); + } + + int refresh_count = 0; +}; + class SplitFormatTrackingTableReader final : public TableReader { public: Status prepare_split(const SplitReadOptions& options) override { @@ -754,6 +766,38 @@ TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { EXPECT_EQ(reader.condition_cache_hit_count(), 8); } +TEST(PaimonHybridReaderTest, ForwardsLatePredicatesToActiveChild) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + paimon::PaimonHybridReader reader; + RefreshTrackingTableReader* child = nullptr; + reader.TEST_set_child_reader_factories( + [&] { + auto tracking = std::make_unique(); + child = tracking.get(); + return tracking; + }, + [] { return std::make_unique(); }); + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_split_format = FileFormat::PARQUET; + split.current_range = make_paimon_native_range(TFileFormatType::FORMAT_PARQUET); + ASSERT_TRUE(reader.prepare_split(split).ok()); + ASSERT_NE(child, nullptr); + ASSERT_TRUE(reader.refresh_conjuncts({}).ok()); + EXPECT_EQ(child->refresh_count, 1); +} + TEST(PaimonHybridReaderTest, NativeCountColumnReportsMetadataRowsThroughHybridReader) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_paimon_hybrid_count_column_test"; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 9615d82b0c36cf..72cee639c44695 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1230,6 +1230,12 @@ class FakeTableReader final : public TableReader { std::shared_ptr state) : _file_schema(std::move(file_schema)), _state(std::move(state)) {} + VExprContextSPtr TEST_mapping_projection(size_t index) const { + DORIS_CHECK(_data_reader.column_mapper != nullptr); + DORIS_CHECK_LT(index, _data_reader.column_mapper->mappings().size()); + return _data_reader.column_mapper->mappings()[index].projection; + } + protected: Status create_file_reader(std::unique_ptr* reader) override { DORIS_CHECK(reader != nullptr); @@ -1669,6 +1675,49 @@ TEST(TableReaderTest, ActiveReaderQueuesRefreshedRuntimeFilterRequest) { ASSERT_TRUE(reader.close().ok()); } +TEST(TableReaderTest, RefreshKeepsActiveMappingProjectionSnapshot) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->eof_with_first_batch = false; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split_options; + split_options.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split_options).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + + const auto active_projection = reader.TEST_mapping_projection(0); + ASSERT_NE(active_projection, nullptr); + ASSERT_TRUE(active_projection->root()->ready_status().ok()); + VExprContextSPtrs refreshed {VExprContext::create_shared( + runtime_filter_wrapper_expr(table_int32_greater_than_expr(0, 0, 1)))}; + ASSERT_TRUE(reader.refresh_conjuncts(std::move(refreshed)).ok()); + + EXPECT_EQ(reader.TEST_mapping_projection(0), active_projection); + EXPECT_TRUE(reader.TEST_mapping_projection(0)->root()->ready_status().ok()); + ASSERT_TRUE(reader.close().ok()); +} + TEST(TableReaderTest, RefreshedConjunctDisablesTableLevelCount) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); From a5fb2e87551f88ac803471cc7281b2e439834eba Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 30 Jul 2026 21:03:40 +0800 Subject: [PATCH 04/10] [cleanup](scan) remove query dictionary filter cache --- be/src/format_v2/parquet/parquet_profile.cpp | 6 - be/src/format_v2/parquet/parquet_profile.h | 6 +- be/src/format_v2/parquet/parquet_scan.cpp | 111 ++---------------- be/src/format_v2/parquet/parquet_scan.h | 4 +- be/src/runtime/query_context.cpp | 8 -- be/src/runtime/query_context.h | 6 - .../runtime/query_dictionary_filter_cache.h | 102 ---------------- .../parquet/parquet_reader_control_test.cpp | 19 --- .../format_v2/parquet/parquet_scan_test.cpp | 24 ++-- 9 files changed, 20 insertions(+), 266 deletions(-) delete mode 100644 be/src/runtime/query_dictionary_filter_cache.h diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 84eb6e0c700066..02b64d7b63c867 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -219,10 +219,6 @@ void ParquetProfile::init(RuntimeProfile* profile) { profile, "DictFilterUnsupportedColumns", TUnit::UNIT, parquet_profile, 1); dict_filter_read_failures = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DictFilterReadFailures", TUnit::UNIT, parquet_profile, 1); - query_dict_filter_cache_hits = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "QueryDictionaryFilterCacheHits", TUnit::UNIT, parquet_profile, 1); - query_dict_filter_cache_misses = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "QueryDictionaryFilterCacheMisses", TUnit::UNIT, parquet_profile, 1); rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", TUnit::UNIT, parquet_profile, 1); bloom_filter_read_time = @@ -367,8 +363,6 @@ ParquetScanProfile ParquetProfile::scan_profile() const { dict_filter_vectorized_runtime_filter_columns, .dict_filter_unsupported_columns = dict_filter_unsupported_columns, .dict_filter_read_failures = dict_filter_read_failures, - .query_dict_filter_cache_hits = query_dict_filter_cache_hits, - .query_dict_filter_cache_misses = query_dict_filter_cache_misses, .rows_filtered_by_dict_filter = rows_filtered_by_dict_filter, .column_reader_profile = column_reader_profile(), }; diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index eb8524430d3c77..2282a70db30540 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -115,9 +115,7 @@ struct ParquetScanProfile { nullptr; // vectorized runtime-filter columns RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; // unsupported columns RuntimeProfile::Counter* dict_filter_read_failures = nullptr; // dictionary read failures - RuntimeProfile::Counter* query_dict_filter_cache_hits = nullptr; - RuntimeProfile::Counter* query_dict_filter_cache_misses = nullptr; - RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; // rows filtered by dict + RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; // rows filtered by dict ParquetColumnReaderProfile column_reader_profile; // nested column read statistics }; @@ -242,8 +240,6 @@ struct ParquetProfile { RuntimeProfile::Counter* dict_filter_vectorized_runtime_filter_columns = nullptr; RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; RuntimeProfile::Counter* dict_filter_read_failures = nullptr; - RuntimeProfile::Counter* query_dict_filter_cache_hits = nullptr; - RuntimeProfile::Counter* query_dict_filter_cache_misses = nullptr; RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; RuntimeProfile::Counter* bloom_filter_read_time = nullptr; }; diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 478735af714d6a..e27ddc8b4c4a68 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -35,7 +35,6 @@ #include "core/column/column_nullable.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" -#include "exec/common/sip_hash.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vcompound_pred.h" #include "exprs/vectorized_fn_call.h" @@ -47,23 +46,12 @@ #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" -#include "runtime/query_context.h" -#include "runtime/query_dictionary_filter_cache.h" #include "runtime/runtime_state.h" #include "util/defer_op.h" -#include "util/hash_util.hpp" #include "util/time.h" namespace doris::format::parquet { -void ParquetScanScheduler::set_runtime_state(RuntimeState* runtime_state) { - _runtime_state = runtime_state; - _query_dictionary_filter_cache = - runtime_state == nullptr || runtime_state->get_query_ctx() == nullptr - ? nullptr - : &runtime_state->get_query_ctx()->query_dictionary_filter_cache(); -} - namespace detail { std::vector order_adaptive_predicates( @@ -1700,63 +1688,6 @@ Status build_dictionary_entry_filter(size_t block_position, return Status::OK(); } -bool contains_mutable_topn_filter(const VExprSPtr& expression) { - if (expression == nullptr) { - return false; - } - if (expression->is_topn_filter()) { - return true; - } - return std::ranges::any_of(expression->children(), contains_mutable_topn_filter); -} - -std::optional dictionary_predicate_digest(size_t block_position, - const VExprContextSPtrs& conjuncts) { - uint64_t digest = 0x6a09e667f3bcc909ULL; - digest = HashUtil::hash64(&block_position, sizeof(block_position), digest); - for (const auto& conjunct : conjuncts) { - if (conjunct == nullptr || contains_mutable_topn_filter(conjunct->root())) { - // TopN's bound tightens in place and is intentionally absent from VExpr::get_digest(). - // Reusing an older bitmap would therefore admit rows beyond the current frontier. - return std::nullopt; - } - digest = conjunct->get_digest(digest); - if (digest == 0) { - return std::nullopt; - } - } - return digest; -} - -std::optional dictionary_filter_cache_key( - size_t block_position, const ParquetColumnSchema& column_schema, - const VExprContextSPtrs& conjuncts, const IColumn& dictionary) { - const auto expression_digest = dictionary_predicate_digest(block_position, conjuncts); - if (!expression_digest.has_value() || - dictionary.size() > std::numeric_limits::max()) { - return std::nullopt; - } - const auto primitive_type = remove_nullable(column_schema.type)->get_primitive_type(); - SipHash dictionary_hash; - const auto type_name = remove_nullable(column_schema.type)->get_name(); - dictionary_hash.update(type_name.data(), type_name.size()); - const uint64_t entries = dictionary.size(); - dictionary_hash.update(entries); - for (size_t entry = 0; entry < dictionary.size(); ++entry) { - dictionary.update_hash_with_value(entry, dictionary_hash); - } - uint64_t hash_low = 0; - uint64_t hash_high = 0; - dictionary_hash.get128(hash_low, hash_high); - return QueryDictionaryFilterCacheKey { - .expression_digest = *expression_digest, - .dictionary_hash_low = hash_low, - .dictionary_hash_high = hash_high, - .dictionary_entries = static_cast(dictionary.size()), - .primitive_type = primitive_type, - }; -} - } // namespace Status ParquetScanScheduler::prepare_current_dictionary_filters( @@ -1839,37 +1770,17 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( OwnedExpressionConjuncts residual_conjuncts; { SCOPED_TIMER(_scan_profile.dict_filter_build_time); - const auto cache_key = dictionary_filter_cache_key( - block_position, *column_schema, conjunct_it->second, *dictionary_values); - std::vector cached_filter; - const bool cache_hit = - cache_key.has_value() && _query_dictionary_filter_cache != nullptr && - _query_dictionary_filter_cache->lookup(*cache_key, &cached_filter); - if (cache_hit) { - dictionary_filter.assign(cached_filter.begin(), cached_filter.end()); - update_counter_if_not_null(_scan_profile.query_dict_filter_cache_hits, 1); - } else { - if (cache_key.has_value() && _query_dictionary_filter_cache != nullptr) { - update_counter_if_not_null(_scan_profile.query_dict_filter_cache_misses, 1); - } - DictionaryEntryFilterKernel filter_kernel = DictionaryEntryFilterKernel::GENERIC; - RETURN_IF_ERROR(build_dictionary_entry_filter( - block_position, *column_schema, conjunct_it->second, *dictionary_values, - &dictionary_filter, &filter_kernel)); - if (filter_kernel == DictionaryEntryFilterKernel::TYPED_FIXED_WIDTH) { - update_counter_if_not_null(_scan_profile.dict_filter_typed_compare_columns, 1); - } else if (filter_kernel == DictionaryEntryFilterKernel::TYPED_STRING) { - update_counter_if_not_null(_scan_profile.dict_filter_string_compare_columns, 1); - } else if (filter_kernel == - DictionaryEntryFilterKernel::VECTORIZED_RUNTIME_FILTER) { - update_counter_if_not_null( - _scan_profile.dict_filter_vectorized_runtime_filter_columns, 1); - } - if (cache_key.has_value() && _query_dictionary_filter_cache != nullptr) { - _query_dictionary_filter_cache->insert( - *cache_key, std::vector(dictionary_filter.begin(), - dictionary_filter.end())); - } + DictionaryEntryFilterKernel filter_kernel = DictionaryEntryFilterKernel::GENERIC; + RETURN_IF_ERROR(build_dictionary_entry_filter(block_position, *column_schema, + conjunct_it->second, *dictionary_values, + &dictionary_filter, &filter_kernel)); + if (filter_kernel == DictionaryEntryFilterKernel::TYPED_FIXED_WIDTH) { + update_counter_if_not_null(_scan_profile.dict_filter_typed_compare_columns, 1); + } else if (filter_kernel == DictionaryEntryFilterKernel::TYPED_STRING) { + update_counter_if_not_null(_scan_profile.dict_filter_string_compare_columns, 1); + } else if (filter_kernel == DictionaryEntryFilterKernel::VECTORIZED_RUNTIME_FILTER) { + update_counter_if_not_null( + _scan_profile.dict_filter_vectorized_runtime_filter_columns, 1); } residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); } diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 19cf0daae4a961..f4c502907165d7 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -42,7 +42,6 @@ class time_zone; namespace doris { class Block; -class QueryDictionaryFilterCache; class RuntimeState; namespace format { @@ -188,7 +187,7 @@ class ParquetScanScheduler { void set_enable_strict_mode(bool enable_strict_mode) { _enable_strict_mode = enable_strict_mode; } - void set_runtime_state(RuntimeState* runtime_state); + void set_runtime_state(RuntimeState* runtime_state) { _runtime_state = runtime_state; } void set_scan_request(std::shared_ptr request); void queue_scan_request(std::shared_ptr request); // Release row-group readers before the owning RuntimeProfile is reported. Native readers @@ -307,7 +306,6 @@ class ParquetScanScheduler { bool _enable_strict_mode = false; bool _enable_bloom_filter = false; RuntimeState* _runtime_state = nullptr; - QueryDictionaryFilterCache* _query_dictionary_filter_cache = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without // reallocating selection indices, dense filter bytes, or compacted-column positions. diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp index e36310fa4879a4..29748d7639d06a 100644 --- a/be/src/runtime/query_context.cpp +++ b/be/src/runtime/query_context.cpp @@ -40,7 +40,6 @@ #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" #include "runtime/memory/heap_profiler.h" -#include "runtime/query_dictionary_filter_cache.h" #include "runtime/runtime_query_statistics_mgr.h" #include "runtime/runtime_state.h" #include "runtime/thread_context.h" @@ -262,13 +261,6 @@ QueryContext::~QueryContext() { elapsed_ms, mem_tracker_msg); } -QueryDictionaryFilterCache& QueryContext::query_dictionary_filter_cache() { - std::call_once(_query_dictionary_filter_cache_once, [this] { - _query_dictionary_filter_cache = std::make_unique(); - }); - return *_query_dictionary_filter_cache; -} - void QueryContext::set_ready_to_execute(Status reason) { set_execution_dependency_ready(); _exec_status.update(reason); diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h index 3964c28c84dfda..3102cc8529613a 100644 --- a/be/src/runtime/query_context.h +++ b/be/src/runtime/query_context.h @@ -50,7 +50,6 @@ class PipelineTask; class QueryTaskController; class Dependency; class RecCTEScanLocalState; -class QueryDictionaryFilterCache; struct ReportStatusRequest { const Status status; @@ -153,8 +152,6 @@ class QueryContext : public std::enable_shared_from_this { return _runtime_predicates.find(source_node_id)->second; } - QueryDictionaryFilterCache& query_dictionary_filter_cache(); - void init_runtime_predicates(const std::vector& topn_filter_descs) { for (auto desc : topn_filter_descs) { _runtime_predicates.try_emplace(desc.source_node_id, desc); @@ -328,9 +325,6 @@ class QueryContext : public std::enable_shared_from_this { void _init_query_mem_tracker(); std::unordered_map _runtime_predicates; - std::once_flag _query_dictionary_filter_cache_once; - std::unique_ptr _query_dictionary_filter_cache; - std::unique_ptr _runtime_filter_mgr; const TQueryOptions _query_options; diff --git a/be/src/runtime/query_dictionary_filter_cache.h b/be/src/runtime/query_dictionary_filter_cache.h deleted file mode 100644 index 83203b64e3f5f8..00000000000000 --- a/be/src/runtime/query_dictionary_filter_cache.h +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "core/data_type/define_primitive_type.h" - -namespace doris { - -struct QueryDictionaryFilterCacheKey { - uint64_t expression_digest = 0; - uint64_t dictionary_hash_low = 0; - uint64_t dictionary_hash_high = 0; - uint32_t dictionary_entries = 0; - PrimitiveType primitive_type = INVALID_TYPE; - - bool operator==(const QueryDictionaryFilterCacheKey&) const = default; -}; - -struct QueryDictionaryFilterCacheKeyHash { - size_t operator()(const QueryDictionaryFilterCacheKey& key) const { - size_t hash = key.expression_digest; - hash ^= key.dictionary_hash_low + 0x9e3779b97f4a7c15ULL + (hash << 6) + (hash >> 2); - hash ^= key.dictionary_hash_high + 0x9e3779b97f4a7c15ULL + (hash << 6) + (hash >> 2); - hash ^= static_cast(key.dictionary_entries) << 1; - hash ^= static_cast(key.primitive_type) << 17; - return hash; - } -}; - -// Query-scoped dictionaries often repeat across row groups and files written by one job. Cache the -// predicate result by dictionary content rather than by file offsets, which are not stable aliases. -class QueryDictionaryFilterCache { -public: - static constexpr size_t DEFAULT_MAX_BYTES = 16 * 1024 * 1024; - - explicit QueryDictionaryFilterCache(size_t max_bytes = DEFAULT_MAX_BYTES) - : _max_bytes(max_bytes) {} - - bool lookup(const QueryDictionaryFilterCacheKey& key, std::vector* result) const { - std::lock_guard lock(_mutex); - const auto it = _entries.find(key); - if (it == _entries.end()) { - return false; - } - *result = it->second; - return true; - } - - bool insert(const QueryDictionaryFilterCacheKey& key, std::vector result) { - const size_t charge = _entry_charge(result.size()); - if (result.empty() || charge > _max_bytes) { - return false; - } - std::lock_guard lock(_mutex); - if (_entries.contains(key)) { - return true; - } - if (charge > _max_bytes - _memory_charge) { - return false; - } - _memory_charge += charge; - _entries.emplace(key, std::move(result)); - return true; - } - -private: - static constexpr size_t _entry_charge(size_t bitmap_bytes) { - // Include fixed entry ownership so many tiny dictionaries cannot bypass the query cap. - return bitmap_bytes + sizeof(QueryDictionaryFilterCacheKey) + sizeof(std::vector) + - 2 * sizeof(void*); - } - - const size_t _max_bytes; - mutable std::mutex _mutex; - size_t _memory_charge = 0; - std::unordered_map, - QueryDictionaryFilterCacheKeyHash> - _entries; -}; - -} // namespace doris diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 0e44569c6a80db..c11fc9706295b3 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -33,7 +33,6 @@ #include "format_v2/parquet/reader/global_rowid_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" #include "format_v2/parquet/selection_vector.h" -#include "runtime/query_dictionary_filter_cache.h" #include "storage/utils.h" namespace doris::format::parquet { @@ -201,24 +200,6 @@ TEST(SelectionVectorTest, BulkCompactionSupportsBothFilterCoordinates) { EXPECT_TRUE(selection.verify(2, 6).ok()); } -TEST(QueryDictionaryFilterCacheTest, ReusesBitmapWithinMemoryLimit) { - QueryDictionaryFilterCache cache(128); - QueryDictionaryFilterCacheKey key {.expression_digest = 11, - .dictionary_hash_low = 22, - .dictionary_hash_high = 33, - .dictionary_entries = 4, - .primitive_type = TYPE_INT}; - std::vector result; - EXPECT_FALSE(cache.lookup(key, &result)); - EXPECT_TRUE(cache.insert(key, {1, 0, 1, 0})); - ASSERT_TRUE(cache.lookup(key, &result)); - EXPECT_EQ(result, std::vector({1, 0, 1, 0})); - - QueryDictionaryFilterCache tiny_cache(2); - EXPECT_FALSE(tiny_cache.insert(key, {1, 0, 1, 0})); - EXPECT_FALSE(tiny_cache.lookup(key, &result)); -} - TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { CursorColumnReader reader; SelectionVector selection(3); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index fccad99fd50936..e842c0b6c72280 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -3179,18 +3179,15 @@ TEST_F(ParquetScanTest, PredicateOnlyDictionaryRangeSkipsTypedValueMaterializati conjunct->close(); } -TEST_F(ParquetScanTest, QueryDictionaryFilterCacheUsesProductionKeysAndBitmaps) { +TEST_F(ParquetScanTest, DictionaryFiltersAreBuiltFromEachReaderSnapshot) { struct ScanResult { std::vector scores; - int64_t cache_hits = 0; - int64_t cache_misses = 0; + int64_t typed_compare_columns = 0; }; - auto query_context = MockQueryContext::create(); auto scan = [&](int32_t lower_bound) { RuntimeProfile profile("profile"); RuntimeState state {TQueryOptions(), TQueryGlobals()}; - state._query_ctx = query_context.get(); auto reader = create_reader(0, -1, &profile); EXPECT_TRUE(reader->init(&state).ok()); @@ -3219,8 +3216,7 @@ TEST_F(ParquetScanTest, QueryDictionaryFilterCacheUsesProductionKeysAndBitmaps) result.scores.push_back(score_column.get_element(row)); } } - result.cache_hits = counter_value(profile, "QueryDictionaryFilterCacheHits"); - result.cache_misses = counter_value(profile, "QueryDictionaryFilterCacheMisses"); + result.typed_compare_columns = counter_value(profile, "DictFilterTypedCompareColumns"); conjunct->close(); EXPECT_TRUE(reader->close().ok()); return result; @@ -3229,24 +3225,20 @@ TEST_F(ParquetScanTest, QueryDictionaryFilterCacheUsesProductionKeysAndBitmaps) write_dictionary_int_pair_parquet_file(_file_path); const auto first = scan(2); EXPECT_EQ(first.scores, std::vector({30, 40, 50, 60})); - EXPECT_EQ(first.cache_hits, 0); - EXPECT_EQ(first.cache_misses, 1); + EXPECT_EQ(first.typed_compare_columns, 1); const auto repeated = scan(2); EXPECT_EQ(repeated.scores, first.scores); - EXPECT_EQ(repeated.cache_hits, 1); - EXPECT_EQ(repeated.cache_misses, 0); + EXPECT_EQ(repeated.typed_compare_columns, 1); const auto changed_predicate = scan(3); EXPECT_EQ(changed_predicate.scores, std::vector({40, 50, 60})); - EXPECT_EQ(changed_predicate.cache_hits, 0); - EXPECT_EQ(changed_predicate.cache_misses, 1); + EXPECT_EQ(changed_predicate.typed_compare_columns, 1); write_dictionary_int_pair_parquet_file(_file_path, {1, 2, 7, 8, 9, 10}); const auto changed_dictionary = scan(2); EXPECT_EQ(changed_dictionary.scores, std::vector({30, 40, 50, 60})); - EXPECT_EQ(changed_dictionary.cache_hits, 0); - EXPECT_EQ(changed_dictionary.cache_misses, 1); + EXPECT_EQ(changed_dictionary.typed_compare_columns, 1); } TEST_F(ParquetScanTest, PredicateOnlyDictionaryTopNUsesDictionaryIds) { @@ -3282,8 +3274,6 @@ TEST_F(ParquetScanTest, PredicateOnlyDictionaryTopNUsesDictionaryIds) { EXPECT_EQ(counter_value(profile, "DictFilterColumns"), 1); EXPECT_EQ(counter_value(profile, "RowsFilteredByDictFilter"), 3); EXPECT_EQ(counter_value(profile, "DictionaryPredicateDirectBatches"), 1); - EXPECT_EQ(counter_value(profile, "QueryDictionaryFilterCacheHits"), 0); - EXPECT_EQ(counter_value(profile, "QueryDictionaryFilterCacheMisses"), 0); prepared.conjunct->close(); } From 909ff1bfb1cd49d1026505c4ce771b1310000f43 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 25 Jul 2026 11:31:55 +0800 Subject: [PATCH 05/10] Revert "[fix](file) Revert split residual predicate ownership (#65998)" This reverts commit b759db26948cebb22206ffa3a5e0a969616f2616. --- be/src/exec/operator/scan_operator.cpp | 29 +- be/src/exec/operator/scan_operator.h | 9 +- be/src/exec/scan/file_scanner_v2.cpp | 145 ++++++-- be/src/exec/scan/file_scanner_v2.h | 33 +- be/src/exec/scan/scanner.cpp | 16 +- be/src/exec/scan/scanner.h | 8 + be/src/format_v2/column_mapper.cpp | 70 +++- be/src/format_v2/column_mapper.h | 15 +- be/src/format_v2/file_reader.h | 4 +- be/src/format_v2/table/hudi_reader.cpp | 22 ++ be/src/format_v2/table/hudi_reader.h | 2 + ...eberg_position_delete_sys_table_reader.cpp | 31 +- be/src/format_v2/table/paimon_reader.cpp | 22 ++ be/src/format_v2/table/paimon_reader.h | 2 + be/src/format_v2/table_reader.cpp | 143 +++++++- be/src/format_v2/table_reader.h | 128 +++++-- .../segment/adaptive_block_size_predictor.cpp | 7 +- .../segment/adaptive_block_size_predictor.h | 1 + be/test/exec/scan/file_scanner_v2_test.cpp | 35 +- .../scan/scanner_late_arrival_rf_test.cpp | 57 ++- ..._position_delete_sys_table_reader_test.cpp | 87 +++++ be/test/format_v2/column_mapper_test.cpp | 68 +++- be/test/format_v2/table/hudi_reader_test.cpp | 147 ++++++++ .../format_v2/table/iceberg_reader_test.cpp | 11 - .../format_v2/table/paimon_reader_test.cpp | 146 ++++++++ be/test/format_v2/table_reader_test.cpp | 340 +++++++++++++++++- .../adaptive_block_size_predictor_test.cpp | 12 + 27 files changed, 1448 insertions(+), 142 deletions(-) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index f945fa0a488810..963758fd40d169 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -73,17 +73,42 @@ bool ScanLocalState::should_run_serial() const { return _parent->cast()._should_run_serial; } -Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state, - int& arrived_rf_num) { +Status ScanLocalStateBase::update_late_arrival_runtime_filter( + RuntimeState* state, int applied_rf_num, int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts) { // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads LockGuard lock(_conjuncts_lock); + arrived_conjuncts.clear(); + size_t conjuncts_before = _conjuncts.size(); RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter(state, _parent->row_descriptor(), arrived_rf_num, _conjuncts)); + if (_conjuncts.size() > conjuncts_before) { + VExprContextSPtrs appended(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + _late_arrival_conjunct_batches.emplace_back(arrived_rf_num, std::move(appended)); + } if (state->enable_adjust_conjunct_order_by_cost()) { std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { return a->execute_cost() < b->execute_cost(); }); }; + // Only re-run partition pruning when try_append_late_arrival_runtime_filter + // actually appended new conjuncts. Otherwise this hook would re-scan all + // partition boundaries on every scheduler pass while there are still + // unapplied RFs (Scanner::_applied_rf_num is not advanced here), wasting + // CPU re-evaluating the same set of RFs against the same boundaries. + if (_conjuncts.size() > conjuncts_before) { + RETURN_IF_ERROR(_on_runtime_filter_update()); + } + for (const auto& [batch_arrived_rf_num, batch] : _late_arrival_conjunct_batches) { + if (batch_arrived_rf_num <= applied_rf_num) { + continue; + } + for (const auto& conjunct : batch) { + VExprContextSPtr cloned; + RETURN_IF_ERROR(conjunct->clone(state, cloned)); + arrived_conjuncts.push_back(std::move(cloned)); + } + } return Status::OK(); } diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index ca9321644f108e..a9a1f3f90f1dd9 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include "common/status.h" #include "common/thread_safety_annotations.h" @@ -91,7 +93,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> { uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } - Status update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num); + Status update_late_arrival_runtime_filter(RuntimeState* state, int applied_rf_num, + int& arrived_rf_num, + VExprContextSPtrs& arrived_conjuncts); Status clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts); @@ -130,6 +134,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> { AnnotatedMutex _conjuncts_lock; RuntimeFilterConsumerHelper _helper; + // Preserve append identity independently of the cost-sorted operator snapshot. Every scanner + // needs the exact RF contexts added since its own applied count. + std::vector> _late_arrival_conjunct_batches; // magic number as seed to generate hash value for condition cache uint64_t _condition_cache_digest = 0; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 39dc99948734b8..8dce2f8cd6a797 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -253,7 +254,9 @@ Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { #ifdef BE_TEST FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader) - : Scanner(state, profile), _table_reader(std::move(table_reader)) {} + : Scanner(state, profile), + _table_reader(std::move(table_reader)), + _scanner_profile(profile) {} Status FileScannerV2::TEST_validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -359,7 +362,9 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); + _initialize_scanner_residual_conjuncts(); auto* profile = _local_state->scanner_profile(); + _scanner_profile = profile; const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); _scanner_total_timer = hierarchy.scanner; _io_timer = hierarchy.io; @@ -393,6 +398,11 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + profile, "ScannerResidualFilterTime", file_scan_profile::SCANNER, 1); + _scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ScannerResidualRowsFiltered", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _refresh_scanner_residual_profile(); SCOPED_TIMER(_scanner_total_timer); SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); @@ -434,6 +444,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e SCOPED_TIMER(_get_block_timer); while (true) { RETURN_IF_CANCELLED(state); + RETURN_IF_ERROR(_sync_table_reader_conjuncts()); if (!_has_prepared_split) { RETURN_IF_ERROR(_prepare_next_split(eof)); if (*eof) { @@ -489,18 +500,33 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } Status FileScannerV2::_filter_output_block(Block* block) { - return _contextualize_output_filter_status(Scanner::_filter_output_block(block), - _get_current_format_type()); -} - -Status FileScannerV2::_contextualize_output_filter_status(Status status, - TFileFormatType::type format_type) { - if (!status.ok() && format_type == TFileFormatType::FORMAT_ORC) { - // Error-preserving expressions cannot be reordered into the ORC reader and therefore run - // at the scanner boundary; keep their error context identical to ORC callback failures. + if (_scanner_residual_conjuncts.empty() || block->rows() == 0) { + return Status::OK(); + } + SCOPED_TIMER(_scanner_residual_filter_timer); + const size_t rows_before_filter = block->rows(); + auto status = VExprContext::filter_block(_scanner_residual_conjuncts, block, block->columns()); + if (!status.ok() && _params != nullptr && + _get_current_format_type() == TFileFormatType::FORMAT_ORC) { status.prepend("Orc row reader nextBatch failed. reason = "); } - return status; + RETURN_IF_ERROR(status); + const int64_t filtered_rows = cast_set(rows_before_filter - block->rows()); + _counter.num_rows_unselected += filtered_rows; + if (_scanner_residual_rows_filtered_counter != nullptr) { + COUNTER_UPDATE(_scanner_residual_rows_filtered_counter, filtered_rows); + } + return Status::OK(); +} + +size_t FileScannerV2::_last_block_rows_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.rows : block.rows(); +} + +size_t FileScannerV2::_last_block_bytes_read(const Block& block) const { + const auto& stats = _table_reader->last_materialized_block_stats(); + return stats.has_materialized_input ? stats.allocated_bytes : block.allocated_bytes(); } Status FileScannerV2::_prepare_next_split(bool* eos) { @@ -587,6 +613,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { RETURN_IF_ERROR(_table_reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(table_conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = const_cast(_params), .io_ctx = _io_ctx, @@ -597,6 +624,9 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .push_down_count_columns = std::move(push_down_count_columns), .condition_cache_digest = _local_state->get_condition_cache_digest(), })); + _table_reader_applied_rf_num = _applied_rf_num; + // RFs collected before TableReader initialization are already present in the full snapshot. + _late_arrival_rf_conjuncts.clear(); return Status::OK(); } @@ -646,15 +676,12 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, std::map partition_values) { format::FileFormat current_split_format; RETURN_IF_ERROR(_to_file_format(get_range_format_type(*_params, range), ¤t_split_format)); - VExprContextSPtrs conjuncts; - RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts)); VExprContextSPtrs partition_prune_conjuncts; if (_state->query_options().enable_runtime_filter_partition_prune) { RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts)); } RETURN_IF_ERROR(_table_reader->prepare_split({ .partition_values = std::move(partition_values), - .conjuncts = std::move(conjuncts), .partition_prune_conjuncts = std::move(partition_prune_conjuncts), // A metadata COUNT split may span scheduler turns. Do not enter that irreversible // synthetic-row path while a runtime filter can still arrive between batches. @@ -841,10 +868,15 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor } Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const { + return _build_table_conjuncts(_conjuncts, conjuncts); +} + +Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const { DORIS_CHECK(conjuncts != nullptr); conjuncts->clear(); - conjuncts->reserve(_conjuncts.size()); - for (const auto& conjunct : _conjuncts) { + conjuncts->reserve(source.size()); + for (const auto& conjunct : source) { VExprSPtr root; RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root)); RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&root, _slot_id_to_global_index)); @@ -854,6 +886,68 @@ Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const return Status::OK(); } +size_t FileScannerV2::_safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + if (!format::TableReader::is_safe_to_pre_execute(conjuncts[conjunct_index])) { + return conjunct_index; + } + } + return conjuncts.size(); +} + +void FileScannerV2::_initialize_scanner_residual_conjuncts() { + _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size(_conjuncts); + // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe + // predicate could run below Scanner before a stateful/error-preserving ordering barrier. + _scanner_residual_conjuncts.assign( + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count), + _conjuncts.end()); + _refresh_scanner_residual_profile(); +} + +void FileScannerV2::_refresh_scanner_residual_profile() { + if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty()) { + return; + } + std::ostringstream predicates; + predicates << "["; + for (size_t conjunct_index = 0; conjunct_index < _scanner_residual_conjuncts.size(); + ++conjunct_index) { + if (conjunct_index > 0) { + predicates << ", "; + } + predicates << _scanner_residual_conjuncts[conjunct_index]->root()->debug_string(); + } + predicates << "]"; + _scanner_profile->add_info_string("ScannerResidualPredicates", predicates.str()); +} + +Status FileScannerV2::_sync_table_reader_conjuncts() { + if (_table_reader == nullptr) { + return Status::OK(); + } + if (_table_reader_applied_rf_num == _applied_rf_num) { + return Status::OK(); + } + VExprContextSPtrs appended; + RETURN_IF_ERROR(_build_table_conjuncts(_late_arrival_rf_conjuncts, &appended)); + const size_t owned_count = _scanner_residual_conjuncts.empty() + ? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts) + : 0; + // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting + // may move a late RF ahead of an old stateful predicate in the full scanner snapshot. + RETURN_IF_ERROR(_table_reader->append_conjuncts_with_ownership(appended, owned_count)); + _table_reader_owned_conjunct_count += owned_count; + _scanner_residual_conjuncts.insert( + _scanner_residual_conjuncts.end(), + _late_arrival_rf_conjuncts.begin() + cast_set(owned_count), + _late_arrival_rf_conjuncts.end()); + _refresh_scanner_residual_profile(); + _late_arrival_rf_conjuncts.clear(); + _table_reader_applied_rf_num = _applied_rf_num; + return Status::OK(); +} + TFileFormatType::type FileScannerV2::_get_current_format_type() const { return get_range_format_type(*_params, _current_range); } @@ -981,17 +1075,19 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) { if (!_should_run_adaptive_batch_size()) { return; } - COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(block.bytes())); - if (block.rows() == 0) { + const auto& stats = _table_reader->last_materialized_block_stats(); + const size_t rows = stats.has_materialized_input ? stats.rows : block.rows(); + const size_t bytes = stats.has_materialized_input ? stats.bytes : block.bytes(); + COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast(bytes)); + if (rows == 0) { return; } - // The sample is taken after TableReader has finalized file-local columns to table columns. - // This matches the memory shape seen by upstream operators and catches very wide nested - // columns, such as map/string payloads, after the first probe batch. + // Residual predicates run after wide table columns are materialized. Learn from that pre-filter + // shape so selective predicates cannot make the next reader batch dangerously large. if (!_block_size_predictor->has_history()) { COUNTER_UPDATE(_adaptive_batch_probe_count_counter, 1); } - _block_size_predictor->update(block); + _block_size_predictor->update(rows, bytes); } Status FileScannerV2::close(RuntimeState* state) { @@ -1183,9 +1279,8 @@ void FileScannerV2::_report_file_reader_predicate_filtered_rows() { const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0; const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows; if (filtered_delta > 0) { - // File readers can evaluate localized conjuncts before a block reaches Scanner. Count - // those rows as scanner-level unselected rows so load statistics stay identical no matter - // whether a predicate is pushed down or evaluated by Scanner::_filter_output_block(). + // FileReader and TableReader both report their owned predicate rows through the shared IO + // context. Preserve scanner-level load statistics without re-evaluating either predicate. _counter.num_rows_unselected += filtered_delta; _reported_predicate_filtered_rows = filtered_rows; } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 87a68e2ab6d176..7992e6ff063be9 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -89,15 +89,22 @@ class FileScannerV2 final : public Scanner { RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics); static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found); static bool TEST_should_skip_empty(const Status& status, bool stopped); - static Status TEST_contextualize_output_filter_status(Status status, - TFileFormatType::type format_type) { - return _contextualize_output_filter_status(std::move(status), format_type); - } static bool TEST_should_run_adaptive_batch_size(bool predictor_initialized, bool current_split_uses_metadata_count) { return _should_run_adaptive_batch_size(predictor_initialized, current_split_uses_metadata_count); } + void TEST_set_scanner_conjuncts(VExprContextSPtrs conjuncts) { + _conjuncts = std::move(conjuncts); + _initialize_scanner_residual_conjuncts(); + } + Status TEST_filter_output_block(Block* block) { return _filter_output_block(block); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } + size_t TEST_scanner_residual_conjunct_count() const { + return _scanner_residual_conjuncts.size(); + } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -116,6 +123,8 @@ class FileScannerV2 final : public Scanner { protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; Status _filter_output_block(Block* block) override; + size_t _last_block_rows_read(const Block& block) const override; + size_t _last_block_bytes_read(const Block& block) const override; void _collect_profile_before_close() override; bool _should_update_load_counters() const override; @@ -134,8 +143,6 @@ class FileScannerV2 final : public Scanner { std::map partition_values); static bool _should_skip_not_found(const Status& status, bool ignore_not_found); static bool _should_skip_empty(const Status& status, bool stopped); - static Status _contextualize_output_filter_status(Status status, - TFileFormatType::type format_type); bool _should_enable_file_meta_cache() const; std::optional _create_global_rowid_context( const TFileRangeDesc& range) const; @@ -147,6 +154,12 @@ class FileScannerV2 final : public Scanner { Status _build_default_expr(const TFileScanSlotInfo& slot_info, VExprContextSPtr* ctx) const; static format::ColumnDefinition _build_table_column(const SlotDescriptor* slot_desc); Status _build_table_conjuncts(VExprContextSPtrs* conjuncts) const; + Status _build_table_conjuncts(const VExprContextSPtrs& source, + VExprContextSPtrs* conjuncts) const; + Status _sync_table_reader_conjuncts(); + static size_t _safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts); + void _initialize_scanner_residual_conjuncts(); + void _refresh_scanner_residual_profile(); static Status _to_file_format(TFileFormatType::type format_type, format::FileFormat* file_format); void _reset_adaptive_batch_size_state(); @@ -183,6 +196,10 @@ class FileScannerV2 final : public Scanner { std::string _current_range_path; std::unique_ptr _table_reader; + size_t _table_reader_owned_conjunct_count = 0; + // Scanner owns one persistent context vector for the first unsafe conjunct and every later + // conjunct. Hybrid child readers may be recreated or switched, but this state must not be. + VExprContextSPtrs _scanner_residual_conjuncts; std::vector _projected_columns; // File formats without embedded schema, such as CSV, still need the FE slot descriptors in // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to @@ -216,6 +233,9 @@ class FileScannerV2 final : public Scanner { RuntimeProfile::Counter* _adaptive_batch_predicted_rows_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_actual_bytes_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr; + RuntimeProfile::Counter* _scanner_residual_filter_timer = nullptr; + RuntimeProfile::Counter* _scanner_residual_rows_filtered_counter = nullptr; + RuntimeProfile* _scanner_profile = nullptr; std::unique_ptr _block_size_predictor; int64_t _reported_predicate_filtered_rows = 0; int64_t _reported_condition_cache_hit_count = 0; @@ -225,6 +245,7 @@ class FileScannerV2 final : public Scanner { int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; int64_t _reported_io_read_time = 0; + int _table_reader_applied_rf_num = 0; }; } // namespace doris diff --git a/be/src/exec/scan/scanner.cpp b/be/src/exec/scan/scanner.cpp index 3069438c764b90..c5a74b358e72da 100644 --- a/be/src/exec/scan/scanner.cpp +++ b/be/src/exec/scan/scanner.cpp @@ -19,6 +19,8 @@ #include +#include + #include "common/config.h" #include "common/status.h" #include "core/block/column_with_type_and_name.h" @@ -145,8 +147,11 @@ Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) { DCHECK(block->rows() == 0); break; } - _num_rows_read += block->rows(); - _num_byte_read += block->allocated_bytes(); + // Some scanners apply owned predicates before returning the block. Account the + // materialized input, not only survivors, so the per-turn progress bound remains + // effective for highly selective predicates. + _num_rows_read += _last_block_rows_read(*block); + _num_byte_read += _last_block_bytes_read(*block); } // 2. Filter the output block finally. @@ -228,7 +233,9 @@ Status Scanner::try_append_late_arrival_runtime_filter() { } DCHECK(_applied_rf_num < _total_rf_num); int arrived_rf_num = 0; - RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter(_state, arrived_rf_num)); + VExprContextSPtrs arrived_conjuncts; + RETURN_IF_ERROR(_local_state->update_late_arrival_runtime_filter( + _state, _applied_rf_num, arrived_rf_num, arrived_conjuncts)); if (arrived_rf_num == _applied_rf_num) { // No newly arrived runtime filters, just return; @@ -238,6 +245,9 @@ Status Scanner::try_append_late_arrival_runtime_filter() { // avoid conjunct destroy in used by storage layer _conjuncts.clear(); RETURN_IF_ERROR(_local_state->clone_conjunct_ctxs(_conjuncts)); + _late_arrival_rf_conjuncts.insert(_late_arrival_rf_conjuncts.end(), + std::make_move_iterator(arrived_conjuncts.begin()), + std::make_move_iterator(arrived_conjuncts.end())); _applied_rf_num = arrived_rf_num; return Status::OK(); } diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index e90754db1c23d6..f12b6b2849f3ae 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -204,6 +204,11 @@ class Scanner { void update_block_avg_bytes(size_t block_avg_bytes) { _block_avg_bytes = block_avg_bytes; } protected: + virtual size_t _last_block_rows_read(const Block& block) const { return block.rows(); } + virtual size_t _last_block_bytes_read(const Block& block) const { + return block.allocated_bytes(); + } + RuntimeState* _state = nullptr; ScanLocalStateBase* _local_state = nullptr; @@ -231,6 +236,9 @@ class Scanner { // Cloned from _conjuncts of scan node. // It includes predicate in SQL and runtime filters. VExprContextSPtrs _conjuncts; + // Exact append-only RF delta for readers that preserve state across multiple splits. It must + // not be reconstructed by position from the cost-sorted full conjunct snapshot. + VExprContextSPtrs _late_arrival_rf_conjuncts; VExprContextSPtrs _projections; // Used in common subexpression elimination to compute intermediate results. std::vector _intermediate_projections; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 7e028370221c38..e0411007b8881d 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -399,6 +399,33 @@ std::string TableColumnMapperOptions::debug_string() const { return out.str(); } +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping) { + if (mapping.table_type == nullptr) { + return false; + } + const auto table_type = remove_nullable(mapping.table_type); + const auto primitive_type = table_type->get_primitive_type(); + if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { + return false; + } + const auto target_len = assert_cast(table_type.get())->len(); + if (target_len <= 0) { + return false; + } + if (mapping.file_type == nullptr) { + return true; + } + const auto file_type = remove_nullable(mapping.file_type); + DORIS_CHECK(file_type != nullptr); + int file_len = -1; + if (file_type->get_primitive_type() == TYPE_VARCHAR || + file_type->get_primitive_type() == TYPE_CHAR || + file_type->get_primitive_type() == TYPE_STRING) { + file_len = assert_cast(file_type.get())->len(); + } + return file_len < 0 || target_len < file_len; +} + std::string ColumnDefinition::debug_string() const { std::ostringstream out; out << "ColumnDefinition{name=" << name << ", identifier=" << field_debug_string(identifier) @@ -2162,7 +2189,8 @@ Status TableColumnMapper::create_scan_request( const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, RuntimeState* runtime_state, - const std::map* fixed_local_positions) { + const std::map* fixed_local_positions, + FilterLocalizationResult* localization_result) { // FileReader evaluates expressions against a file-local block. This mapper owns the // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); @@ -2204,7 +2232,8 @@ Status TableColumnMapper::create_scan_request( // 2. Build referenced predicate columns // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); - RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + RETURN_IF_ERROR( + localize_filters(table_filters, file_request, runtime_state, localization_result)); for (const auto& mapping : _hidden_mappings) { if (!mapping.file_local_id.has_value()) { continue; @@ -2218,9 +2247,9 @@ Status TableColumnMapper::create_scan_request( if (is_visible_output) { continue; } - // File-local filtering is an optimization; Scanner still evaluates the original - // table-level conjunct after TableReader returns. Only truly hidden mappings are absent - // from that scanner-visible block and may safely discard their payload here. + // A localized predicate is enforced exactly before TableReader materializes output. Only + // truly hidden mappings are absent from the final table block and may discard their + // payload after that file-local evaluation. if (std::ranges::any_of(file_request->predicate_columns, [local_id](const LocalColumnIndex& projection) { return projection.column_id() == local_id; @@ -2269,7 +2298,11 @@ ColumnMapping* TableColumnMapper::_find_filter_mapping(GlobalIndex global_index) Status TableColumnMapper::localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state) { + RuntimeState* runtime_state, + FilterLocalizationResult* localization_result) { + if (localization_result != nullptr) { + localization_result->localized_filters.assign(table_filters.size(), false); + } std::set localized_predicate_columns; FilterProjectionMap filter_projections; auto filter_mappings = _filter_visible_mappings(); @@ -2307,17 +2340,28 @@ Status TableColumnMapper::localize_filters(const std::vector& table // This keeps expression localization independent from filter iteration order. filter_mappings = _filter_visible_mappings(); const auto global_to_file_slot = build_file_slot_rewrite_map(filter_mappings, _filter_entries); - for (const auto& table_filter : table_filters) { + for (size_t filter_index = 0; filter_index < table_filters.size(); ++filter_index) { + const auto& table_filter = table_filters[filter_index]; if (table_filter.conjunct != nullptr && table_filter.conjunct->root() != nullptr) { const auto root = table_filter.conjunct->root(); const auto impl = root->get_impl(); const auto predicate = impl != nullptr ? impl : root; - if (!predicate->is_deterministic() || + if (!table_filter.can_localize || !predicate->is_deterministic() || !table_filter_has_only_local_entries(table_filter, _filter_entries)) { continue; } - // Scanner evaluates the original conjunct after final materialization. Only predicates - // whose result is stable across repeated execution may also run as a file-local copy. + if (runtime_state != nullptr && + runtime_state->query_options().truncate_char_or_varchar_columns && + std::ranges::any_of(table_filter.global_indices, [&](GlobalIndex global_index) { + const auto* mapping = _find_filter_mapping(global_index); + return mapping != nullptr && requires_char_or_varchar_truncation(*mapping); + })) { + // The table predicate observes the bounded value after finalize; evaluating it on + // a wider file string would change equality and range semantics. + continue; + } + // FileReader becomes the exact owner only for a stable predicate whose complete + // expression can be rewritten against this split's physical schema. RewriteContext rewrite_context {.runtime_state = runtime_state}; VExprSPtr rewrite_root; Status clone_status; @@ -2328,8 +2372,7 @@ Status TableColumnMapper::localize_filters(const std::vector& table // `element_at(MAP_VALUES(m)[1], 'age') > 30`. The current file-local rewrite only // understands top-level slots and struct-element paths rooted at top-level slots; // cloning such expressions can hit the generic TExpr complex-type limitation. - // Leave them above TableReader, where Scanner evaluates the original table-level - // conjunct after final materialization. + // Leave them for TableReader after final table-schema materialization. #ifndef NDEBUG return Status::InternalError( "Failed to clone table filter for file-local rewrite: {}, expr={}", @@ -2365,6 +2408,9 @@ Status TableColumnMapper::localize_filters(const std::vector& table auto localized_conjunct = VExprContext::create_shared(std::move(localized_root)); RETURN_IF_ERROR(rewrite_context.prepare_created_exprs(localized_conjunct.get())); file_request->conjuncts.push_back(std::move(localized_conjunct)); + if (localization_result != nullptr) { + localization_result->localized_filters[filter_index] = true; + } for (const auto global_index : table_filter.global_indices) { const auto* mapping = _find_filter_mapping(global_index); if (mapping != nullptr && mapping->file_local_id.has_value() && diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index 68d0ff357c0e52..a6aaa342ad8cb9 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -40,6 +40,13 @@ namespace doris::format { struct ColumnDefinition; struct TableFilter; +// Reports which table filters were fully rewritten into exact file-local predicates for the +// current split. The result is aligned with the TableFilter input vector and must not be reused for +// another split because schema evolution can change localization independently for every file. +struct FilterLocalizationResult { + std::vector localized_filters; +}; + enum class TableColumnMappingMode { // Match by ColumnDefinition::identifier TYPE_INT as field id. BY_FIELD_ID, @@ -165,6 +172,8 @@ struct TableColumnMapperOptions { std::string debug_string() const; }; +bool requires_char_or_varchar_truncation(const ColumnMapping& mapping); + Status clone_table_expr_tree(const VExprSPtr& expr, VExprSPtr* cloned_expr); const Field* find_partition_value(const ColumnDefinition& table_column, const std::map& partition_values); @@ -197,7 +206,8 @@ class TableColumnMapper { const std::vector& table_filters, const std::vector& projected_columns, FileScanRequest* file_request, RuntimeState* runtime_state = nullptr, - const std::map* fixed_local_positions = nullptr); + const std::map* fixed_local_positions = nullptr, + FilterLocalizationResult* localization_result = nullptr); // Localize table-level filters to the file schema. // Trivial mappings can copy structured predicates directly. Type changes may be localized with @@ -205,7 +215,8 @@ class TableColumnMapper { // table-level finalize/filter fallback. virtual Status localize_filters(const std::vector& table_filters, FileScanRequest* file_request, - RuntimeState* runtime_state = nullptr); + RuntimeState* runtime_state = nullptr, + FilterLocalizationResult* localization_result = nullptr); void clear() { _mappings.clear(); _hidden_mappings.clear(); diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 65a2d03417c605..017792e42f7473 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -77,7 +77,9 @@ struct FileScanRequest { std::vector predicate_only_columns; // file-local column id -> file-local output block position. std::map local_positions; - // Row-level filters converted to file-local expressions from table-level predicates. + // Row-level filters converted to file-local expressions from table-level predicates. Readers + // must enforce these exactly on returned rows; metadata pruning alone does not transfer + // predicate ownership away from TableReader. VExprContextSPtrs conjuncts; // Delete predicates converted to file-local expressions. A TRUE result means that the row is // deleted, so readers must invert each result when building their keep filter. diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 838d98b5e52e14..20b64e2ca93f0e 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -137,6 +137,27 @@ void HudiHybridReader::set_batch_size(size_t batch_size) { } } +Status HudiHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& HudiHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + int64_t HudiHybridReader::condition_cache_hit_count() const { // Keep the wrapper count cumulative across native/JNI dispatch so scanner-level delta // accounting neither loses a child hit nor observes a counter reset on a split switch. @@ -188,6 +209,7 @@ Status HudiHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index c06e1b238b62c4..53893b74a24c8c 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -66,6 +66,8 @@ class HudiHybridReader final : public format::TableReader { Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; int64_t condition_cache_hit_count() const override; #ifdef BE_TEST diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index b2b37c1bf09226..4b65f745b08ec5 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -163,6 +163,12 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( const format::SplitReadOptions& options) { RETURN_IF_ERROR(close()); RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + if (current_split_pruned()) { + return Status::OK(); + } + // This synthetic reader has no physical schema where a predicate can be localized, so every + // split predicate must run after its system-table columns have been materialized. + RETURN_IF_ERROR(_prepare_all_conjuncts_as_remaining()); // The inner delete-file reader has distinct counters, so the outer preparation can safely // contain its cache miss/open work without re-entering the same RuntimeProfile timer. SCOPED_TIMER(_profile.total_timer); @@ -175,6 +181,7 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); + _reset_materialized_block_stats(); DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(block->columns() == _projected_columns.size()); @@ -192,9 +199,19 @@ Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) return Status::OK(); } - size_t read_rows = 0; if (_delete_file_kind == DeleteFileKind::DELETION_VECTOR) { - return _append_deletion_vector_block(block, &read_rows, eos); + size_t read_rows = 0; + RETURN_IF_ERROR(_append_deletion_vector_block(block, &read_rows, eos)); + if (read_rows > 0) { + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + } + if (read_rows == 0) { + // Yield after one deletion-vector batch so cancellation and Scanner row budgets are + // observed even when residual predicates reject every synthesized row. + block->clear_column_data(_projected_columns.size()); + } + return Status::OK(); } DORIS_CHECK(_position_reader != nullptr); @@ -208,8 +225,18 @@ Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) RETURN_IF_ERROR(_position_reader->get_block(&delete_block, &position_reader_eof)); const size_t delete_rows = delete_block.rows(); if (delete_rows > 0) { + size_t read_rows = 0; RETURN_IF_ERROR( _append_position_delete_block(block, delete_block, delete_rows, &read_rows)); + _record_materialized_block_stats(*block, read_rows); + RETURN_IF_ERROR(_filter_remaining_conjuncts(block, &read_rows)); + if (read_rows == 0) { + // A filtered materialized batch is still progress; return it to Scanner instead of + // consuming an unbounded number of position-delete batches in this call. + block->clear_column_data(_projected_columns.size()); + *eos = false; + return Status::OK(); + } *eos = false; return Status::OK(); } diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 93183af178babb..d4252a833c2b6e 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -172,6 +172,27 @@ void PaimonHybridReader::set_batch_size(size_t batch_size) { } } +Status PaimonHybridReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + // The wrapper snapshot initializes future children, while every existing child needs the same + // late RF immediately so active and later reused splits keep identical predicate ownership. + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + RETURN_IF_ERROR(format::TableReader::append_conjuncts(conjuncts)); + if (_native_reader != nullptr) { + RETURN_IF_ERROR(_native_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + if (_jni_reader != nullptr) { + RETURN_IF_ERROR(_jni_reader->append_conjuncts_with_ownership(conjuncts, owned_count)); + } + return Status::OK(); +} + +const format::MaterializedBlockStats& PaimonHybridReader::last_materialized_block_stats() const { + // FileScannerV2 budgets cooperative work from the child that actually materialized the block. + return _current_split_reader != nullptr ? _current_split_reader->last_materialized_block_stats() + : format::TableReader::last_materialized_block_stats(); +} + int64_t PaimonHybridReader::condition_cache_hit_count() const { // Both children survive split switches, so the wrapper must publish their cumulative totals; // returning only the active child would make FileScannerV2's monotonic delta go backwards. @@ -229,6 +250,7 @@ Status PaimonHybridReader::_init_child_reader(format::TableReader* reader, RETURN_IF_ERROR(reader->init({ .projected_columns = _projected_columns, .conjuncts = std::move(conjuncts), + .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count, .format = file_format, .scan_params = _scan_params, .io_ctx = _io_ctx, diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index 8570f2efba624e..981b57bc1be24f 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -72,6 +72,8 @@ class PaimonHybridReader final : public format::TableReader { Status abort_split() override; Status close() override; void set_batch_size(size_t batch_size) override; + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override; + const format::MaterializedBlockStats& last_materialized_block_stats() const override; int64_t condition_cache_hit_count() const override; #ifdef BE_TEST diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index c542cb908f53d7..619d36cdd8ad6a 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -680,6 +680,8 @@ Status TableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PrepareSplitTime", table_profile, 1); _profile.finalize_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "FinalizeBlockTime", table_profile, 1); + _profile.residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "ResidualFilterTime", table_profile, 1); _profile.create_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CreateReaderTime", table_profile, 1); _profile.pushdown_agg_timer = @@ -723,6 +725,9 @@ Status TableReader::init(TableReadOptions&& options) { _push_down_count_columns = options.push_down_count_columns; _initial_condition_cache_digest = options.condition_cache_digest; _condition_cache_digest = _initial_condition_cache_digest; + _table_reader_owned_conjunct_count = + options.table_reader_owned_conjunct_count.value_or(options.conjuncts.size()); + DORIS_CHECK_LE(_table_reader_owned_conjunct_count, options.conjuncts.size()); _projected_columns = std::move(options.projected_columns); if (supports_iceberg_scan_semantics_v1(_scan_params)) { for (auto& projected_column : _projected_columns) { @@ -736,7 +741,64 @@ Status TableReader::init(TableReadOptions&& options) { } _system_properties = create_system_properties(_scan_params); _mapper_options.mode = TableColumnMappingMode::BY_NAME; - _conjuncts = std::move(options.conjuncts); + return _replace_conjuncts(options.conjuncts); +} + +Status TableReader::_prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared) { + DORIS_CHECK(source != nullptr); + DORIS_CHECK(source->root() != nullptr); + DORIS_CHECK(prepared != nullptr); + VExprSPtr root; + RETURN_IF_ERROR(clone_table_expr_tree(source->root(), &root)); + auto conjunct = VExprContext::create_shared(std::move(root)); + RETURN_IF_ERROR(conjunct->prepare(_runtime_state, RowDescriptor {})); + RETURN_IF_ERROR(conjunct->open(_runtime_state)); + *prepared = std::move(conjunct); + return Status::OK(); +} + +Status TableReader::_replace_conjuncts(const VExprContextSPtrs& conjuncts) { + VExprContextSPtrs prepared; + prepared.reserve(conjuncts.size()); + for (const auto& source : conjuncts) { + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + prepared.push_back(std::move(conjunct)); + } + _conjuncts = std::move(prepared); + return Status::OK(); +} + +Status TableReader::append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count) { + DORIS_CHECK(!_appended_table_reader_owned_conjunct_count.has_value()); + _appended_table_reader_owned_conjunct_count = table_reader_owned_conjunct_count; + auto status = append_conjuncts(conjuncts); + _appended_table_reader_owned_conjunct_count.reset(); + return status; +} + +Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { + const size_t owned_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + DORIS_CHECK_LE(owned_count, conjuncts.size()); + // Once Scanner owns a suffix, later predicates cannot be inserted into the TableReader-owned + // prefix without reordering them ahead of that stateful/error-preserving barrier. + DORIS_CHECK(owned_count == 0 || _table_reader_owned_conjunct_count == _conjuncts.size()); + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { + const auto& source = conjuncts[conjunct_index]; + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + _conjuncts.push_back(conjunct); + if (conjunct_index < owned_count) { + ++_table_reader_owned_conjunct_count; + } + if (_current_task != nullptr && conjunct_index < owned_count) { + // The active reader has already fixed its localized predicate set. Appended runtime + // filters must remain residual until the next split rebuilds its FileScanRequest. + _remaining_conjuncts.push_back(std::move(conjunct)); + } + } return Status::OK(); } @@ -744,20 +806,27 @@ Status TableReader::_build_table_filters_from_conjuncts() { _table_filters.clear(); _constant_pruning_safe_filter_count = 0; bool in_safe_prefix = true; - for (const auto& conjunct : _conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < _conjuncts.size(); ++conjunct_index) { + const auto& conjunct = _conjuncts[conjunct_index]; DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); // `_table_filters` omits expressions without slot references, but such an expression still // occupies a position in the row-level conjunct order. Record how many localized filters // precede the first unsafe original conjunct so constant pruning cannot jump over a - // slotless non-deterministic/error-preserving barrier. Unsafe predicates remain solely on - // Scanner's original row-level path because localizing a clone would execute their state - // twice with independent state. - if (in_safe_prefix && !_is_safe_to_pre_execute(conjunct)) { + // slotless non-deterministic/error-preserving barrier. An unsafe predicate is either kept + // on TableReader's post-materialization path by a standalone caller or carried only for + // analysis when FileScannerV2 owns the ordered suffix. + if (in_safe_prefix && !is_safe_to_pre_execute(conjunct)) { in_safe_prefix = false; } + const size_t filters_before = _table_filters.size(); RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); + for (size_t filter_index = filters_before; filter_index < _table_filters.size(); + ++filter_index) { + _table_filters[filter_index].source_conjunct_index = conjunct_index; + _table_filters[filter_index].can_localize = in_safe_prefix; + } if (in_safe_prefix) { _constant_pruning_safe_filter_count = _table_filters.size(); } @@ -863,6 +932,59 @@ Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { return Status::OK(); } +Status TableReader::_prepare_all_conjuncts_as_remaining() { + // Expression contexts carry mutable state (for example sequence/stateful functions). Select + // from the TableReader-owned contexts instead of reopening clones for every split. + _remaining_conjuncts.assign( + _conjuncts.begin(), + _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count)); + return Status::OK(); +} + +Status TableReader::_prepare_remaining_conjuncts( + const FilterLocalizationResult& localization_result) { + DORIS_CHECK(localization_result.localized_filters.size() == _table_filters.size()); + std::vector localized_conjuncts(_conjuncts.size(), false); + for (size_t filter_index = 0; filter_index < _table_filters.size(); ++filter_index) { + if (!localization_result.localized_filters[filter_index]) { + continue; + } + const size_t source_index = _table_filters[filter_index].source_conjunct_index; + DORIS_CHECK(source_index < localized_conjuncts.size()); + localized_conjuncts[source_index] = true; + } + + _remaining_conjuncts.clear(); + for (size_t conjunct_index = 0; conjunct_index < _table_reader_owned_conjunct_count; + ++conjunct_index) { + if (localized_conjuncts[conjunct_index]) { + continue; + } + _remaining_conjuncts.push_back(_conjuncts[conjunct_index]); + } + return Status::OK(); +} + +Status TableReader::_filter_remaining_conjuncts(Block* block, size_t* rows) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(rows != nullptr); + if (*rows == 0 || _remaining_conjuncts.empty()) { + return Status::OK(); + } + SCOPED_TIMER(_profile.residual_filter_timer); + const size_t rows_before_filter = *rows; + auto status = VExprContext::filter_block(_remaining_conjuncts, block, block->columns()); + if (!status.ok() && _format == FileFormat::ORC) { + status.prepend("Orc row reader nextBatch failed. reason = "); + } + RETURN_IF_ERROR(status); + *rows = block->columns() == 0 ? rows_before_filter : block->rows(); + if (_io_ctx != nullptr) { + _io_ctx->predicate_filtered_rows += rows_before_filter - *rows; + } + return Status::OK(); +} + Status TableReader::_open_local_filter_exprs(const FileScanRequest& file_request) { RowDescriptor row_desc; for (const auto& conjunct : file_request.conjuncts) { @@ -1103,6 +1225,9 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); _current_split_pruned = false; + // Predicate localization belongs to the physical schema of one split. Clear the previous + // ownership before any early return so a pruned or failed split cannot leak it to the next one. + _remaining_conjuncts.clear(); _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; _condition_cache_digest_covers_current_split = options.condition_cache_digest.has_value(); if (options.condition_cache_digest.has_value()) { @@ -1116,7 +1241,7 @@ Status TableReader::prepare_split(const SplitReadOptions& options) { _condition_cache_digest = _initial_condition_cache_digest; } if (options.conjuncts.has_value()) { - _conjuncts = *options.conjuncts; + RETURN_IF_ERROR(_replace_conjuncts(*options.conjuncts)); } // Update to current split format to handle ORC/PARQUET files in one table. _format = options.current_split_format; @@ -1186,7 +1311,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& // Keep only the safe prefix of the original conjunct order. If an unsafe conjunct is // skipped, a later predicate could prune the split before the unsafe one reaches its // normal row-level evaluation point. - if (!_is_safe_to_pre_execute(conjunct)) { + if (!is_safe_to_pre_execute(conjunct)) { break; } std::set global_indices; @@ -1223,7 +1348,7 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& can_filter_all); } -bool TableReader::_is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { +bool TableReader::is_safe_to_pre_execute(const VExprContextSPtr& conjunct) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); const auto root = conjunct->root(); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index ad081eba9f3499..65d88765510499 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -78,10 +78,14 @@ namespace doris::format { using DeleteRows = std::vector; // Row-level predicates on table/global schema. They are rewritten to file-local expressions when -// possible, and remain the source of row-level filtering after localization. +// possible; otherwise TableReader evaluates them after final table-schema materialization. struct TableFilter { VExprContextSPtr conjunct; std::vector global_indices; + size_t source_conjunct_index = 0; + // False after the first unsafe source conjunct so file-local execution cannot reorder a later + // predicate ahead of stateful or error-preserving table semantics. + bool can_localize = true; }; struct ScanTask { @@ -113,6 +117,7 @@ struct ReadProfile { RuntimeProfile::Counter* exec_timer = nullptr; RuntimeProfile::Counter* prepare_split_timer = nullptr; RuntimeProfile::Counter* finalize_timer = nullptr; + RuntimeProfile::Counter* residual_filter_timer = nullptr; RuntimeProfile::Counter* create_reader_timer = nullptr; RuntimeProfile::Counter* pushdown_agg_timer = nullptr; RuntimeProfile::Counter* open_reader_timer = nullptr; @@ -129,12 +134,23 @@ struct ReadProfile { RuntimeProfile::Counter* file_reader_close_timer = nullptr; }; +struct MaterializedBlockStats { + bool has_materialized_input = false; + size_t rows = 0; + size_t bytes = 0; + size_t allocated_bytes = 0; +}; + struct TableReadOptions { // Columns need to be read from file and output by table reader. They are all in table/global // schema semantics. const std::vector projected_columns; // All complex conjuncts from scan operator const VExprContextSPtrs conjuncts; + // Number of leading conjuncts whose row-level execution is owned by TableReader/FileReader. + // FileScannerV2 still passes the complete ordered list so mapping, pruning guards, aggregate + // eligibility, and condition-cache analysis see the exact query semantics. nullopt means all. + const std::optional table_reader_owned_conjunct_count = std::nullopt; // File format of the underlying data files, needed for reader initialization and reader-level // filter pushdown. const FileFormat format; @@ -207,6 +223,10 @@ class TableReader { #ifdef BE_TEST size_t TEST_batch_size() const { return _batch_size; } + size_t TEST_conjunct_count() const { return _conjuncts.size(); } + size_t TEST_table_reader_owned_conjunct_count() const { + return _table_reader_owned_conjunct_count; + } void TEST_set_condition_cache_hit_count(int64_t hits) { _condition_cache_hit_count = hits; } bool TEST_current_data_file_is_immutable() const { DORIS_CHECK(_current_task != nullptr); @@ -232,6 +252,25 @@ class TableReader { return _current_split_uses_metadata_count; } + // Runtime filters that arrive after a split has opened cannot be pushed into that file reader. + // Keep their expression contexts in TableReader and evaluate them as residual predicates for + // the active reader; later splits can localize them normally. + virtual Status append_conjuncts(const VExprContextSPtrs& conjuncts); + + // Append a full ordered snapshot delta while marking only its leading prefix as owned by + // TableReader/FileReader. This non-virtual wrapper preserves the long-standing virtual API and + // carries the ownership boundary through hybrid readers to their children. + Status append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, + size_t table_reader_owned_conjunct_count); + + // Shared safety classification for deciding which ordered conjunct prefix may execute below + // Scanner without changing stateful or error-preserving semantics. + static bool is_safe_to_pre_execute(const VExprContextSPtr& conjunct); + + virtual const MaterializedBlockStats& last_materialized_block_stats() const { + return _last_materialized_block_stats; + } + // Discard the active split after the caller decides an error is ignorable, for example a // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. @@ -251,6 +290,7 @@ class TableReader { _remaining_file_level_count = -1; _current_split_uses_metadata_count = false; _current_split_pruned = false; + _remaining_conjuncts.clear(); return Status::OK(); } @@ -260,6 +300,7 @@ class TableReader { virtual Status get_block(Block* block, bool* eos) { SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); + _last_materialized_block_stats = {}; DORIS_CHECK(block->columns() == _projected_columns.size()); block->clear_column_data(_projected_columns.size()); @@ -332,7 +373,7 @@ class TableReader { RETURN_IF_ERROR(_check_file_block_columns("after file reader get_block", current_rows)); #endif DORIS_CHECK(block->columns() == _data_reader.column_mapper->mappings().size()); - RETURN_IF_ERROR(finalize_chunk(block, current_rows)); + RETURN_IF_ERROR(finalize_chunk(block, ¤t_rows)); #ifndef NDEBUG RETURN_IF_ERROR( _check_table_block_columns("after finalize_chunk", block, current_rows)); @@ -341,6 +382,13 @@ class TableReader { _current_reader_reached_eof = !stopped_during_read; RETURN_IF_ERROR(close_current_reader()); } + if (current_rows == 0) { + // One materialized batch is one Scanner progress unit even when residual + // predicates reject every row. Returning here preserves row-budget and + // cancellation checks in Scanner::get_block(). + block->clear_column_data(_projected_columns.size()); + return Status::OK(); + } return Status::OK(); } } @@ -358,6 +406,7 @@ class TableReader { _remaining_table_level_count = -1; _remaining_file_level_count = -1; _current_split_uses_metadata_count = false; + _remaining_conjuncts.clear(); return Status::OK(); } @@ -443,14 +492,17 @@ class TableReader { // reader with the request. File scan request carries row-level expression filters and // file-level pruning hints. Only expression filters decide returned rows. auto file_request = std::make_shared(); + FilterLocalizationResult localization_result; RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( - _table_filters, _projected_columns, file_request.get(), _runtime_state)); + _table_filters, _projected_columns, file_request.get(), _runtime_state, + &localization_result)); bool constant_filter_pruned_split = false; RETURN_IF_ERROR(_evaluate_constant_filters(&constant_filter_pruned_split)); if (constant_filter_pruned_split) { RETURN_IF_ERROR(close_current_reader()); return Status::OK(); } + RETURN_IF_ERROR(_prepare_remaining_conjuncts(localization_result)); // COUNT(*) has no semantic column argument, but Nereids retains a minimum-width scan slot // so the scan node still has an output tuple. Record only the current non-predicate file // columns before table-format hooks add row-position or equality-delete dependencies. This @@ -528,9 +580,13 @@ class TableReader { } Status _build_table_filters_from_conjuncts(); + Status _replace_conjuncts(const VExprContextSPtrs& conjuncts); + Status _prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared); + Status _prepare_remaining_conjuncts(const FilterLocalizationResult& localization_result); + Status _prepare_all_conjuncts_as_remaining(); + Status _filter_remaining_conjuncts(Block* block, size_t* rows); Status _evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, bool* can_filter_all); - static bool _is_safe_to_pre_execute(const VExprContextSPtr& conjunct); Status _build_partition_prune_block(Block* block) const; Status _open_local_filter_exprs(const FileScanRequest& file_request); Status _init_reader_condition_cache(const FileScanRequest& file_request); @@ -549,7 +605,7 @@ class TableReader { if (table_filter.conjunct == nullptr) { continue; } - DORIS_CHECK(_is_safe_to_pre_execute(table_filter.conjunct)); + DORIS_CHECK(is_safe_to_pre_execute(table_filter.conjunct)); // RuntimeFilterExpr does not implement execute_column_impl(); it is evaluated by the // row-level filter path through execute_filter(). Constant split pruning uses // VExprContext::execute() on a one-row synthetic block, so runtime filters must not be @@ -766,6 +822,7 @@ class TableReader { } _table_filters.clear(); _constant_pruning_safe_filter_count = 0; + _remaining_conjuncts.clear(); _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); _data_reader.block_template.clear(); @@ -782,15 +839,28 @@ class TableReader { } } + void _reset_materialized_block_stats() { _last_materialized_block_stats = {}; } + + void _record_materialized_block_stats(const Block& block, size_t rows) { + _last_materialized_block_stats = { + .has_materialized_input = true, + .rows = rows, + .bytes = block.bytes(), + .allocated_bytes = block.allocated_bytes(), + }; + } + // Finalize file-local block to table/global schema block. - Status finalize_chunk(Block* block, const size_t rows) { + Status finalize_chunk(Block* block, size_t* rows) { + DORIS_CHECK(rows != nullptr); SCOPED_TIMER(_profile.finalize_timer); size_t idx = 0; const auto& mappings = _data_reader.column_mapper->mappings(); for (const auto& mapping : mappings) { ColumnPtr column; - RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, - &column, idx + 1 == mappings.size())); + RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, + *rows, &column, + idx + 1 == mappings.size())); block->replace_by_position(idx, IColumn::mutate(std::move(column))); idx++; } @@ -798,7 +868,12 @@ class TableReader { // Enforce CHAR/VARCHAR length declared by the table schema after all file-to-table // materialization has finished. RETURN_IF_ERROR(_truncate_char_or_varchar_columns(block)); - return Status::OK(); + // Preserve the cost of materialization before residual predicates shrink the block. The + // scanner uses this snapshot for bounded progress and adaptive batch sizing. + _record_materialized_block_stats(*block, *rows); + // Predicate ownership is split-local: only predicates not acknowledged as exact by this + // split's FileScanRequest run here, after virtual/default/schema-evolution values exist. + return _filter_remaining_conjuncts(block, rows); } // Materialize virtual columns in the table block, such as Iceberg _row_id and @@ -962,31 +1037,7 @@ class TableReader { // - table VARCHAR(10), file STRING: truncate to 10 because STRING has no declared bound; // - table STRING, any file type: no truncation because the target has no bound. static bool _should_truncate_char_or_varchar_column(const ColumnMapping& mapping) { - if (mapping.table_type == nullptr) { - return false; - } - const auto table_type = remove_nullable(mapping.table_type); - const auto primitive_type = table_type->get_primitive_type(); - if (primitive_type != TYPE_VARCHAR && primitive_type != TYPE_CHAR) { - return false; - } - const auto target_len = assert_cast(table_type.get())->len(); - if (target_len <= 0) { - return false; - } - if (mapping.file_type == nullptr) { - return true; - } - const auto file_type = remove_nullable(mapping.file_type); - DORIS_CHECK(file_type != nullptr); - int file_len = -1; - if (file_type->get_primitive_type() == TYPE_VARCHAR || - file_type->get_primitive_type() == TYPE_CHAR || - file_type->get_primitive_type() == TYPE_STRING) { - file_len = assert_cast(file_type.get())->len(); - } - - return file_len < 0 || target_len < file_len; + return requires_char_or_varchar_truncation(mapping); } // Truncate a materialized CHAR/VARCHAR column in place by reusing the vectorized substring @@ -1083,9 +1134,8 @@ class TableReader { if (!_all_runtime_filters_applied_for_split) { return false; } - // Scanner owns the original conjunct list and evaluates it after TableReader finalizes - // rows. Even a slotless conjunct that cannot become a TableFilter must see every source - // row before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. + // Even a slotless conjunct that cannot become a TableFilter must see every source row + // before an aggregate reduces the stream to synthetic COUNT/MINMAX rows. if (!_conjuncts.empty()) { return false; } @@ -1906,6 +1956,10 @@ class TableReader { // intentionally absent from that vector but must still act as ordering barriers. size_t _constant_pruning_safe_filter_count = 0; VExprContextSPtrs _conjuncts; + size_t _table_reader_owned_conjunct_count = 0; + std::optional _appended_table_reader_owned_conjunct_count; + VExprContextSPtrs _remaining_conjuncts; + MaterializedBlockStats _last_materialized_block_stats; ReadProfile _profile; // Parsed from row-position based delete files, including position delete and deletion vector. DeleteRows* _delete_rows = nullptr; diff --git a/be/src/storage/segment/adaptive_block_size_predictor.cpp b/be/src/storage/segment/adaptive_block_size_predictor.cpp index d8cc700f579853..7a5ad573a2ea6c 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.cpp +++ b/be/src/storage/segment/adaptive_block_size_predictor.cpp @@ -32,11 +32,14 @@ AdaptiveBlockSizePredictor::AdaptiveBlockSizePredictor(size_t preferred_block_si _metadata_hint_bytes_per_row(metadata_hint_bytes_per_row) {} void AdaptiveBlockSizePredictor::update(const Block& block) { - size_t rows = block.rows(); + update(block.rows(), block.bytes()); +} + +void AdaptiveBlockSizePredictor::update(size_t rows, size_t bytes) { if (rows == 0) { return; } - double cur = static_cast(block.bytes()) / static_cast(rows); + double cur = static_cast(bytes) / static_cast(rows); if (!_has_history) { _bytes_per_row = cur; diff --git a/be/src/storage/segment/adaptive_block_size_predictor.h b/be/src/storage/segment/adaptive_block_size_predictor.h index e03f18c2a536d2..f327fec8517f63 100644 --- a/be/src/storage/segment/adaptive_block_size_predictor.h +++ b/be/src/storage/segment/adaptive_block_size_predictor.h @@ -60,6 +60,7 @@ class AdaptiveBlockSizePredictor { // Update EWMA estimates from a completed batch. Must be called only when block.rows() > 0 // and the batch returned Status::OK(). void update(const Block& block); + void update(size_t rows, size_t bytes); // Predict how many rows the next batch should read. // Never exceeds |block_size_rows|; never returns less than 1. diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 2950fa4911d9e4..3bc0da7b89082b 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -666,18 +666,6 @@ TEST(FileScannerV2Test, EndOfFileIsSkippedAsEmptySplit) { EXPECT_FALSE(FileScannerV2::TEST_should_skip_empty(Status::OK(), false)); } -TEST(FileScannerV2Test, OrcScannerResidualFilterRetainsNextBatchContext) { - auto status = FileScannerV2::TEST_contextualize_output_filter_status( - Status::InvalidArgument("synthetic row filter failure"), TFileFormatType::FORMAT_ORC); - EXPECT_NE(status.to_string().find("nextBatch failed"), std::string::npos) << status; - EXPECT_NE(status.to_string().find("synthetic row filter failure"), std::string::npos) << status; - - status = FileScannerV2::TEST_contextualize_output_filter_status( - Status::InvalidArgument("synthetic row filter failure"), - TFileFormatType::FORMAT_PARQUET); - EXPECT_EQ(status.to_string().find("nextBatch failed"), std::string::npos) << status; -} - // Scenario: partition slots are identified from the explicit FE category when present, otherwise // from the legacy is_file_slot flag. Scanner-generated rowid columns must never be treated as // partition columns even if FE marks them as non-file slots. @@ -804,4 +792,27 @@ TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { EXPECT_EQ(partition_conjuncts[0], conjuncts[0]); } +TEST(FileScannerV2Test, ScannerOwnsUnsafeConjunctAndOrderedSuffixInProfile) { + const auto bool_type = std::make_shared(); + auto unsafe_predicate = std::make_shared(); + unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part")); + VExprContextSPtrs conjuncts { + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 1), + runtime_filter_context(std::move(unsafe_predicate), 2), + runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 3), + }; + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("file_scanner_v2"); + FileScannerV2 scanner(&state, &profile, nullptr); + scanner.TEST_set_scanner_conjuncts(std::move(conjuncts)); + + EXPECT_EQ(scanner.TEST_table_reader_owned_conjunct_count(), 1); + EXPECT_EQ(scanner.TEST_scanner_residual_conjunct_count(), 2); + const auto* residual_predicates = profile.get_info_string("ScannerResidualPredicates"); + ASSERT_NE(residual_predicates, nullptr); + EXPECT_FALSE(residual_predicates->empty()); + EXPECT_NE(residual_predicates->find("SlotRef"), std::string::npos) << *residual_predicates; +} + } // namespace doris diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index c65f3bd6f5ce07..deb754d499b192 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -68,6 +68,23 @@ class TestScanner final : public Scanner { std::list _blocks; }; +class HighCostPredicate final : public VExpr { +public: + HighCostPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + double execute_cost() const override { return 100.0; } + +private: + const std::string _expr_name = "high_cost_stateful_predicate"; +}; + class ScannerLateArrivalRfTest : public RuntimeFilterTest { public: void SetUp() override { @@ -86,6 +103,7 @@ class ScannerLateArrivalRfTest : public RuntimeFilterTest { // the counter advances after RFs arrive and that the second call short-circuits // via the fast path at the top of the function. TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { + _runtime_states[0]->_query_options.__set_enable_adjust_conjunct_order_by_cost(true); std::vector rf_descs = { TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build(), TRuntimeFilterDescBuilder().add_planId_to_target_expr(0).build()}; @@ -107,26 +125,55 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { auto local_state = std::make_shared(_runtime_states[0].get(), op.get()); + auto initial_conjunct = VExprContext::create_shared(std::make_shared()); + ASSERT_TRUE(initial_conjunct->prepare(_runtime_states[0].get(), row_desc).ok()); + ASSERT_TRUE(initial_conjunct->open(_runtime_states[0].get()).ok()); + local_state->_conjuncts.push_back(initial_conjunct); + std::vector> rf_dependencies; ASSERT_TRUE(local_state->_helper.init(_runtime_states[0].get(), true, 0, 0, rf_dependencies, "") .ok()); + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + local_state->_helper._consumers[0]->signal(producer.get()); + ASSERT_TRUE(local_state->_helper + .acquire_runtime_filter(_runtime_states[0].get(), local_state->_conjuncts, + row_desc) + .ok()); + ASSERT_EQ(local_state->_conjuncts.size(), 2); + auto scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), -1 /*limit*/, &_profile); - ASSERT_TRUE(scanner->init(_runtime_states[0].get(), {}).ok()); + ASSERT_TRUE(scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); + auto second_scanner = std::make_unique(_runtime_states[0].get(), local_state.get(), + -1 /*limit*/, &_profile); + ASSERT_TRUE(second_scanner->init(_runtime_states[0].get(), local_state->_conjuncts).ok()); ASSERT_EQ(scanner->_total_rf_num, 2); ASSERT_EQ(scanner->_applied_rf_num, 0); - std::shared_ptr producer; - ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), rf_descs.data(), &producer).ok()); - producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); - local_state->_helper._consumers[0]->signal(producer.get()); local_state->_helper._consumers[1]->signal(producer.get()); // First call after both RFs arrived: counter must advance to total. Before // the fix this stayed at 0 because the assignment was missing. ASSERT_TRUE(scanner->try_append_late_arrival_runtime_filter().ok()); ASSERT_EQ(scanner->_applied_rf_num, 2); + ASSERT_EQ(scanner->_late_arrival_rf_conjuncts.size(), 1); + for (const auto& conjunct : scanner->_late_arrival_rf_conjuncts) { + EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); + } + ASSERT_EQ(scanner->_conjuncts.size(), 3); + EXPECT_EQ(scanner->_conjuncts.back()->expr_name(), "high_cost_stateful_predicate"); + + // The first scanner consumes the shared helper's expression, so another scanner can only get + // the exact delta from the local state's append-only RF batch history. + ASSERT_TRUE(second_scanner->try_append_late_arrival_runtime_filter().ok()); + ASSERT_EQ(second_scanner->_applied_rf_num, 2); + ASSERT_EQ(second_scanner->_late_arrival_rf_conjuncts.size(), 1); + EXPECT_NE(dynamic_cast( + second_scanner->_late_arrival_rf_conjuncts[0]->root().get()), + nullptr); // Second call: must hit the fast-path early return without re-cloning. // We clear `_conjuncts` and verify the function does NOT repopulate them; diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index ee041bab621785..ea836ed7aab70e 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -29,11 +29,14 @@ #include "core/column/column_nullable.h" #include "core/column/column_string.h" #include "core/column/column_struct.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exprs/vexpr.h" +#include "format/orc/orc_memory_stream_test.h" +#include "format/table/iceberg_scan_semantics.h" #include "format/table/parquet_utils.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "io/io_common.h" @@ -74,6 +77,31 @@ class FillColumnsTrackingReader final : public GenericReader { } }; +class RejectAllRowsPredicate final : public VExpr { +public: + RejectAllRowsPredicate() : VExpr(std::make_shared(), false) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 0); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(); + return Status::OK(); + } + +private: + const std::string _name = "RejectAllRowsPredicate"; +}; + SlotDescriptor* make_slot(ObjectPool* pool, int id, std::string name, DataTypePtr type) { TSlotDescriptor slot_desc; slot_desc.__set_id(id); @@ -645,4 +673,63 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, StopsBeforeExpandingDeletionVect EXPECT_TRUE(eof); } +TEST(IcebergPositionDeleteSysTableV2ReaderTest, + AllFilteredDeletionVectorYieldsBeforeObservingCancellation) { + ObjectPool pool; + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("test_profile"); + const auto nullable_int64 = make_nullable(std::make_shared()); + std::vector file_slot_descs { + make_slot(&pool, 0, "pos", nullable_int64), + }; + + auto conjunct = VExprContext::create_shared(std::make_shared()); + RowDescriptor row_desc; + ASSERT_TRUE(conjunct->prepare(&state, row_desc).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + + format::iceberg::IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._io_ctx = std::make_shared(); + reader._file_slot_descs = &file_slot_descs; + reader._projected_columns.resize(file_slot_descs.size()); + reader._remaining_conjuncts = {conjunct}; + reader._has_split = true; + reader._delete_file_kind = + format::iceberg::IcebergPositionDeleteSysTableV2Reader::DeleteFileKind::DELETION_VECTOR; + reader._batch_size = 1; + reader._dv_positions.add(uint64_t {7}); + reader._dv_positions.add(uint64_t {9}); + reader._dv_positions.add(uint64_t {11}); + reader._next_dv_position.emplace(reader._dv_positions.begin()); + + Block block = make_output_block(file_slot_descs); + bool eof = false; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_FALSE(eof); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); + + reader._io_ctx->should_stop = true; + ASSERT_TRUE(reader.get_block(&block, &eof).ok()); + EXPECT_TRUE(eof); + ASSERT_TRUE(reader._next_dv_position.has_value()); + EXPECT_EQ(**reader._next_dv_position, 9); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET, + "parquet"); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, OrcRowUsesAnyFieldIdMapping) { + run_mixed_id_position_delete_test(format::FileFormat::ORC, TFileFormatType::FORMAT_ORC, "orc"); +} + +TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetReadsNestedIdlessWrapper) { + run_v2_nested_wrapper_position_delete_test(); +} + } // namespace doris diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index 29d4efbe17f44e..d3428b6134a479 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -2337,6 +2337,39 @@ TEST(ColumnMapperLocalizeFiltersTest, VisibleLocalFilterAddsPredicateColumnAndCo EXPECT_TRUE(localized_slot->data_type()->equals(*int_type)); } +TEST(ColumnMapperLocalizeFiltersTest, ReportsLocalizationForEachSplitMapping) { + const auto int_type = i32(); + auto table_column = name_col("id", int_type); + const std::vector table_schema = {table_column}; + TableFilter filter { + .conjunct = VExprContext::create_shared(int_gt(table_slot(0, 0, int_type, "id"), 1)), + .global_indices = {GlobalIndex(0)}}; + + TableColumnMapper local_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(local_mapper.create_mapping(table_schema, {}, {name_col("id", int_type, 7)}).ok()); + FileScanRequest local_request; + FilterLocalizationResult local_result; + ASSERT_TRUE(local_mapper + .create_scan_request({filter}, table_schema, &local_request, nullptr, + &local_result) + .ok()); + ASSERT_EQ(local_result.localized_filters.size(), 1); + EXPECT_TRUE(local_result.localized_filters[0]); + ASSERT_EQ(local_request.conjuncts.size(), 1); + + TableColumnMapper missing_mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(missing_mapper.create_mapping(table_schema, {}, {}).ok()); + FileScanRequest missing_request; + FilterLocalizationResult missing_result; + ASSERT_TRUE(missing_mapper + .create_scan_request({filter}, table_schema, &missing_request, nullptr, + &missing_result) + .ok()); + ASSERT_EQ(missing_result.localized_filters.size(), 1); + EXPECT_FALSE(missing_result.localized_filters[0]); + EXPECT_TRUE(missing_request.conjuncts.empty()); +} + TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { const auto binary_type = varbinary(); const auto table_column = name_col("partition_key", binary_type); @@ -2362,6 +2395,35 @@ TEST(ColumnMapperLocalizeFiltersTest, VarbinaryFilterStaysAboveFileReader) { EXPECT_TRUE(request.conjuncts.empty()); } +TEST(ColumnMapperLocalizeFiltersTest, VarcharWidthTruncationFilterStaysAboveFileReader) { + const auto table_type = std::make_shared(3, TYPE_VARCHAR); + const auto file_type = std::make_shared(10, TYPE_VARCHAR); + const auto table_column = name_col("value", table_type); + const auto file_column = name_col("value", file_type, 7); + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_NAME}); + ASSERT_TRUE(mapper.create_mapping({table_column}, {}, {file_column}).ok()); + + TableFilter filter {.conjunct = VExprContext::create_shared(binary_predicate( + TExprOpcode::EQ, table_slot(0, 0, table_type, "value"), + literal(table_type, Field::create_field("abc")))), + .global_indices = {GlobalIndex(0)}}; + TQueryOptions query_options; + query_options.__set_truncate_char_or_varchar_columns(true); + RuntimeState state {query_options, TQueryGlobals()}; + FileScanRequest request; + FilterLocalizationResult localization_result; + + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request, &state, + &localization_result) + .ok()); + ASSERT_EQ(localization_result.localized_filters.size(), 1); + EXPECT_FALSE(localization_result.localized_filters[0]); + EXPECT_TRUE(request.conjuncts.empty()); + ASSERT_EQ(request.non_predicate_columns.size(), 1); + EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(7)); +} + TEST(ColumnMapperLocalizeFiltersTest, NestedVarbinaryFilterStaysAboveFileReader) { const auto table_column = struct_name_col( "payload", {name_col("id", i32()), name_col("binary_value", varbinary())}); @@ -2512,7 +2574,7 @@ TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); } -TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsPayloadForScannerBoundary) { +TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsOutputPayload) { const auto int_type = i32(); auto quantity = name_col("ss_quantity", int_type); auto tax = name_col("ss_ext_tax", int_type); @@ -2537,8 +2599,8 @@ TEST(ColumnMapperScanRequestTest, OrdinaryPredicateSlotRetainsPayloadForScannerB EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.non_predicate_columns.size(), 1); EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(1)); - // The scanner evaluates its table-level conjuncts after TableReader returns, so a visible - // predicate slot cannot be replaced with a default-valued placeholder at the file boundary. + // A visible predicate slot is still part of the table output and cannot be replaced with a + // default-valued placeholder after file-local filtering. EXPECT_TRUE(request.predicate_only_columns.empty()); } diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index e75eee47be39d6..283cee43f5018b 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -37,6 +37,9 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format_v2/column_data.h" #include "gen_cpp/ExternalTableSchema_types.h" #include "gen_cpp/PlanNodes_types.h" @@ -153,6 +156,54 @@ class RefreshTrackingTableReader final : public TableReader { int refresh_count = 0; }; +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -274,6 +325,102 @@ TEST(HudiHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 123); } +TEST(HudiHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + hudi::HudiHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(HudiHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(HudiHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("hudi_scanner_stateful_residual"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + run_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); + run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + TEST(HudiHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { hudi::HudiHybridReader reader; reader.TEST_install_batch_size_children(); diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 782463e335a743..c42d68933f2014 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -1146,11 +1146,6 @@ VExprContextSPtr prepared_conjunct(RuntimeState* state, const VExprSPtr& expr) { return ctx; } -void apply_final_conjuncts(Block* block, const VExprContextSPtrs& conjuncts) { - const auto status = VExprContext::filter_block(conjuncts, block, block->columns()); - ASSERT_TRUE(status.ok()) << status; -} - TEST(IcebergV2ReaderTest, IcebergVirtualColumnsUseRowLineageMetadata) { const auto test_dir = std::filesystem::temp_directory_path() / "doris_iceberg_virtual_columns_test"; @@ -1389,9 +1384,6 @@ TEST(IcebergV2ReaderTest, IcebergRowIdPredicateFiltersAfterRowLineageMaterializa bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); @@ -1443,9 +1435,6 @@ TEST(IcebergV2ReaderTest, IcebergLastUpdatedSequencePredicateFiltersAfterMateria bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); - ASSERT_EQ(block.rows(), 3); - - apply_final_conjuncts(&block, conjuncts); ASSERT_EQ(block.rows(), 1); expect_nullable_int64_column_values(*block.get_by_position(0).column, {1001}); expect_nullable_int64_column_values(*block.get_by_position(1).column, {77}); diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 32b82ab12acbe3..41180443419aad 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -45,6 +45,9 @@ #include "core/data_type/data_type_string.h" #include "core/field.h" #include "exec/common/endian.h" +#include "exec/scan/file_scanner_v2.h" +#include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "format/format_common.h" #include "format/table/deletion_vector_reader.h" #include "format/table/paimon_reader.h" @@ -101,6 +104,54 @@ class SplitFormatTrackingTableReader final : public TableReader { FileFormat prepared_format = FileFormat::JNI; }; +class AppendTrackingTableReader final : public TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts += conjuncts.size(); + owned_conjuncts += _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); + return Status::OK(); + } + + size_t appended_conjuncts = 0; + size_t owned_conjuncts = 0; +}; + +class OneRowTableReader final : public TableReader { +public: + Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } + + Status get_block(Block* block, bool* eos) override { + auto column = ColumnInt32::create(); + column->insert_value(1); + block->replace_by_position(0, std::move(column)); + *eos = false; + return Status::OK(); + } +}; + +class StatefulHybridPredicate final : public VExpr { +public: + explicit StatefulHybridPredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + _observed_invocations->push_back(_invocation++); + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_constant() const override { return false; } + bool is_deterministic() const override { return false; } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulHybridPredicate"; +}; + DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -759,6 +810,101 @@ TEST(PaimonHybridReaderTest, AdaptiveBatchSizeReachesBothChildReaders) { EXPECT_EQ(child_batch_sizes.second, 321); } +TEST(PaimonHybridReaderTest, ReportsActiveChildMaterializedBlockStats) { + paimon::PaimonHybridReader reader; + reader.TEST_install_batch_size_children(); + reader._current_split_reader = reader._native_reader.get(); + reader._native_reader->_last_materialized_block_stats = { + .has_materialized_input = true, .rows = 7, .bytes = 70, .allocated_bytes = 96}; + + const auto& stats = reader.last_materialized_block_stats(); + EXPECT_TRUE(stats.has_materialized_input); + EXPECT_EQ(stats.rows, 7); + EXPECT_EQ(stats.bytes, 70); + EXPECT_EQ(stats.allocated_bytes, 96); +} + +TEST(PaimonHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + auto native_reader = std::make_unique(); + auto jni_reader = std::make_unique(); + auto* native_reader_ptr = native_reader.get(); + auto* jni_reader_ptr = jni_reader.get(); + reader._native_reader = std::move(native_reader); + reader._jni_reader = std::move(jni_reader); + + auto literal = VLiteral::create_shared(std::make_shared(), + Field::create_field(1)); + ASSERT_TRUE(reader.append_conjuncts_with_ownership( + {VExprContext::create_shared(std::move(literal))}, 0) + .ok()); + EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); +} + +TEST(PaimonHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("paimon_scanner_stateful_residual"); + auto scan_params = make_local_parquet_scan_params(); + auto hybrid_reader = std::make_unique(); + auto* hybrid_reader_ptr = hybrid_reader.get(); + hybrid_reader_ptr->TEST_set_child_reader_factories( + [] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + std::vector observed_invocations; + auto conjunct = VExprContext::create_shared( + std::make_shared(&observed_invocations)); + ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); + ASSERT_TRUE(conjunct->open(&state).ok()); + FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); + scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); + + const std::vector projected_columns { + make_table_column(0, "id", std::make_shared()), + }; + ASSERT_TRUE(hybrid_reader_ptr + ->init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + auto run_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); + ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); + }; + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + run_split(FileFormat::JNI, make_paimon_jni_range()); + run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + + EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); +} + TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { paimon::PaimonHybridReader reader; reader.TEST_install_batch_size_children(); diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 72cee639c44695..a7243f08ba5f85 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -297,6 +297,37 @@ class NonDeterministicPartitionPredicate final : public VExpr { const std::string _expr_name = "NonDeterministicPartitionPredicate"; }; +class StatefulSequencePredicate final : public VExpr { +public: + explicit StatefulSequencePredicate(std::vector* observed_invocations) + : VExpr(std::make_shared(), false), + _observed_invocations(observed_invocations) {} + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(_observed_invocations != nullptr); + _observed_invocations->push_back(_invocation++); + auto result = ColumnUInt8::create(); + result->get_data().resize_fill(count, 1); + result_column = std::move(result); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_observed_invocations); + return Status::OK(); + } + +private: + std::vector* const _observed_invocations; + mutable int _invocation = 0; + const std::string _expr_name = "StatefulSequencePredicate"; +}; + class NullableArrayBigintDefaultExpr final : public VExpr { public: explicit NullableArrayBigintDefaultExpr(DataTypePtr data_type) @@ -534,6 +565,23 @@ void write_parquet_file(const std::string& file_path, int32_t id, const std::str builder.build())); } +void write_single_int_parquet_file(const std::string& file_path, const std::string& column_name, + int32_t value) { + auto schema = arrow::schema({arrow::field(column_name, arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array({value})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, + builder.build())); +} + void write_struct_parquet_file(const std::string& file_path, int32_t id) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false)}); arrow::StructBuilder builder( @@ -1060,6 +1108,8 @@ struct FakeFileReaderState { bool stop_during_aggregate = false; bool stop_during_read = false; bool not_found_during_init = false; + int batch_count = 1; + int get_block_count = 0; std::shared_ptr last_request; std::shared_ptr pending_request; std::optional last_aggregate_request; @@ -1099,7 +1149,7 @@ class FakeFileReader final : public FileReader { RETURN_IF_ERROR(FileReader::open(std::move(request))); _state->last_request = _request; ++_state->open_count; - _returned_batch = false; + _returned_batches = 0; return Status::OK(); } @@ -1116,7 +1166,8 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); DORIS_CHECK(_request != nullptr); - if (_returned_batch) { + ++_state->get_block_count; + if (_returned_batches >= _state->batch_count) { *rows = 0; *eof = true; return Status::OK(); @@ -1163,9 +1214,9 @@ class FakeFileReader final : public FileReader { DORIS_CHECK(_state->io_ctx != nullptr); _state->io_ctx->should_stop = true; } - _returned_batch = true; + ++_returned_batches; *rows = 2; - *eof = _state->eof_with_first_batch; + *eof = _state->eof_with_first_batch && _returned_batches >= _state->batch_count; if (_state->condition_cache_ctx != nullptr && !_state->condition_cache_ctx->is_hit && _state->condition_cache_ctx->filter_result != nullptr && !_state->condition_cache_ctx->filter_result->empty()) { @@ -1221,7 +1272,7 @@ class FakeFileReader final : public FileReader { private: std::vector _schema; std::shared_ptr _state; - bool _returned_batch = false; + int _returned_batches = 0; }; class FakeTableReader final : public TableReader { @@ -1434,13 +1485,54 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_FALSE(predicate_executed); + EXPECT_TRUE(predicate_executed); EXPECT_FALSE(eos); + // The file was still opened, proving constant pruning did not jump over the unsafe predicate; + // the predicate is evaluated only after the resulting table row is materialized. EXPECT_EQ(fake_state->open_count, 1); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, UnsafePredicateRunsAfterTableMaterialization) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto unsafe_predicate = + std::make_shared(&predicate_executed); + unsafe_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_TRUE(predicate_executed); ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath) { +TEST(TableReaderTest, ScannerOwnedUnsafePredicateIsPassedButNotExecutedByTableReader) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); std::vector projected_columns; @@ -1457,6 +1549,7 @@ TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath) { ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .table_reader_owned_conjunct_count = 0, .format = FileFormat::PARQUET, .scan_params = nullptr, .io_ctx = nullptr, @@ -1471,9 +1564,178 @@ TEST(TableReaderTest, UnsafePredicateStaysOnScannerPath) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_NE(fake_state->last_request, nullptr); EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + EXPECT_EQ(reader.TEST_conjunct_count(), 1); + EXPECT_EQ(reader.TEST_table_reader_owned_conjunct_count(), 0); EXPECT_FALSE(predicate_executed); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualExpressionStateSurvivesAcrossSplits) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + std::vector observed_invocations; + auto stateful_predicate = std::make_shared(&observed_invocations); + stateful_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct(&state, stateful_predicate)}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + for (int split_index = 0; split_index < 2; ++split_index) { + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_EQ(block.rows(), 2); + ASSERT_TRUE(reader.close().ok()); + } + + EXPECT_EQ(observed_invocations, std::vector({0, 1})); +} + +TEST(TableReaderTest, AllFilteredResidualReturnsAfterOneMaterializedBatch) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + bool predicate_executed = false; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); + EXPECT_EQ(fake_state->get_block_count, 1); + EXPECT_EQ(fake_state->close_count, 0); + EXPECT_TRUE(reader.last_materialized_block_stats().has_materialized_input); + EXPECT_EQ(reader.last_materialized_block_stats().rows, 2); + EXPECT_GT(reader.last_materialized_block_stats().bytes, 0); + EXPECT_GT(reader.last_materialized_block_stats().allocated_bytes, 0); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, LateConjunctFiltersAlreadyOpenSplit) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto fake_state = std::make_shared(); + fake_state->batch_count = 2; + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_EQ(block.rows(), 2); + + bool predicate_executed = false; + auto late_predicate = std::make_shared(&predicate_executed); + late_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + ASSERT_TRUE( + reader.append_conjuncts({VExprContext::create_shared(std::move(late_predicate))}).ok()); + block = build_table_block(projected_columns); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); + EXPECT_EQ(fake_state->get_block_count, 2); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(TableReaderTest, ResidualFilteringHasDedicatedProfileTimer) { + std::vector file_schema; + file_schema.push_back(make_file_column(0, "id", std::make_shared())); + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("scanner"); + bool predicate_executed = false; + auto fake_state = std::make_shared(); + FakeTableReader reader(file_schema, fake_state); + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, + std::make_shared( + &predicate_executed))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + }) + .ok()); + + SplitReadOptions split; + split.current_range.__set_path("fake-table-reader-input"); + ASSERT_TRUE(reader.prepare_split(split).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + + EXPECT_TRUE(predicate_executed); + ASSERT_NE(profile.get_counter("ResidualFilterTime"), nullptr); + EXPECT_GT(profile.get_counter("ResidualFilterTime")->value(), 0); ASSERT_TRUE(reader.close().ok()); } @@ -1524,9 +1786,11 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { EXPECT_EQ(fake_state->open_count, 1); ASSERT_NE(fake_state->last_request, nullptr); // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. - // The later predicate must stay on the scanner's row-level path instead of running inside the + // The later predicate must stay on the post-materialization path instead of running inside the // file reader before the unsafe conjunct. EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); ASSERT_TRUE(reader.close().ok()); } @@ -1901,8 +2165,13 @@ TEST(TableReaderTest, SlotlessConjunctDisablesAggregatePushdown) { // presence still prevents the fake aggregate count (3) from replacing the two physical rows. ASSERT_NE(fake_state->last_request, nullptr); EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); - EXPECT_EQ(block.rows(), 2); + // The two physical rows are then filtered at the table boundary, where slotless predicates are + // evaluated exactly even though they cannot be localized to a file column. + EXPECT_EQ(block.rows(), 0); + EXPECT_FALSE(eos); EXPECT_TRUE(predicate_executed); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); ASSERT_TRUE(reader.close().ok()); } @@ -4971,6 +5240,59 @@ TEST(TableReaderTest, VExprPredicateSurvivesReopenSplit) { std::filesystem::remove_all(test_dir); } +TEST(TableReaderTest, RecomputesPredicateExecutionLayerForEverySplit) { + const auto test_dir = std::filesystem::temp_directory_path() / + "doris_table_reader_split_local_predicate_test"; + std::filesystem::remove_all(test_dir); + std::filesystem::create_directories(test_dir); + + const auto local_file = (test_dir / "local.parquet").string(); + const auto missing_file = (test_dir / "missing.parquet").string(); + write_single_int_parquet_file(local_file, "id", 3); + write_single_int_parquet_file(missing_file, "other", 9); + + std::vector projected_columns; + projected_columns.push_back(make_table_column(0, "id", std::make_shared())); + set_name_identifiers(&projected_columns); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = projected_columns, + .conjuncts = {prepared_conjunct( + &state, table_int32_greater_than_expr(0, 0, 2))}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + + ASSERT_TRUE(reader.prepare_split(build_split_options(local_file)).ok()); + Block block = build_table_block(projected_columns); + bool eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + ASSERT_FALSE(eos); + expect_int32_column_values(*block.get_by_position(0).column, {3}); + ASSERT_TRUE(reader.close().ok()); + + // The same predicate cannot be file-local when this split omits `id`. It must be rebuilt as a + // table-level predicate over the materialized NULL instead of inheriting the previous split's + // file-local ownership or escaping without exact evaluation. + ASSERT_TRUE(reader.prepare_split(build_split_options(missing_file)).ok()); + block = build_table_block(projected_columns); + eos = false; + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(eos); + EXPECT_EQ(block.rows(), 0); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_TRUE(eos); + + ASSERT_TRUE(reader.close().ok()); + std::filesystem::remove_all(test_dir); +} + TEST(TableReaderTest, CreateScanRequestDeduplicatesSharedPredicateColumns) { const auto int_type = std::make_shared(); const std::vector projected_columns = { diff --git a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp index 60b6f37b8ceeba..64795dace19a14 100644 --- a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp +++ b/be/test/storage/segment/adaptive_block_size_predictor_test.cpp @@ -89,6 +89,18 @@ TEST_F(AdaptiveBlockSizePredictorTest, NoHistoryReturnsMaxRows) { EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), expected_bpr); } +TEST_F(AdaptiveBlockSizePredictorTest, ExplicitMaterializedSampleUsesPreFilterShape) { + AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0); + + // Callers that filter a block before returning it can still report the rows and bytes that + // were actually materialized upstream. + pred.update(32, 32 * 4096); + + EXPECT_TRUE(pred.has_history_for_test()); + EXPECT_DOUBLE_EQ(pred.bytes_per_row_for_test(), 4096.0); + EXPECT_EQ(pred.predict_next_rows(), 2048); +} + // ── Test 2: EWMA convergence ────────────────────────────────────────────────── // When every update delivers the same sample, the EWMA stays exactly at that // value (0.9*v + 0.1*v == v for any v). From ca93eff002d9040bf116e9d5f3cc3d970a20bec8 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 25 Jul 2026 13:35:12 +0800 Subject: [PATCH 06/10] [fix](be) Restore FileScannerV2 residual predicate ownership ### What problem does this PR solve? Issue Number: N/A Related PR: #65998 Problem Summary: Cost sorting could move a late runtime filter ahead of an older unsafe predicate when FileScannerV2 rebuilt partition-pruning predicates for the next split. TableReader also prepared and opened Scanner-owned predicate suffixes even though it retained them only for pruning analysis. Preserve append identity separately from the cost-sorted Scanner snapshot, and clone Scanner-owned TableReader predicates without creating duplicate execution state. ### Release note None ### Check List (For Author) - Test: Unit Test - FileScannerV2Test.*, TableReaderTest.*, and ScannerLateArrivalRfTest.* - Behavior changed: Yes. Late runtime filters retain predicate ordering barriers, and Scanner-owned predicates have a single execution-state owner. - Does this need documentation: No --- be/src/exec/scan/file_scanner_v2.cpp | 23 ++++- be/src/exec/scan/file_scanner_v2.h | 3 + be/src/format_v2/table_reader.cpp | 30 +++++-- be/src/format_v2/table_reader.h | 1 + be/test/exec/scan/file_scanner_v2_test.cpp | 51 +++++++++++ be/test/format_v2/table_reader_test.cpp | 100 +++++++++++++++++++++ 6 files changed, 199 insertions(+), 9 deletions(-) diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index 8dce2f8cd6a797..bc42ffc7c1ae0e 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -593,6 +593,21 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { RETURN_IF_ERROR(_to_file_format(format_type, &file_format)); DORIS_CHECK(_table_reader != nullptr); + if (!_late_arrival_rf_conjuncts.empty()) { + const size_t owned_count = _scanner_residual_conjuncts.empty() + ? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts) + : 0; + _table_reader_owned_conjunct_count += owned_count; + _scanner_residual_conjuncts.insert( + _scanner_residual_conjuncts.end(), + _late_arrival_rf_conjuncts.begin() + cast_set(owned_count), + _late_arrival_rf_conjuncts.end()); + _append_ordered_conjuncts.insert(_append_ordered_conjuncts.end(), + _late_arrival_rf_conjuncts.begin(), + _late_arrival_rf_conjuncts.end()); + _late_arrival_rf_conjuncts.clear(); + _refresh_scanner_residual_profile(); + } VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); std::optional> push_down_count_columns; @@ -625,8 +640,6 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { .condition_cache_digest = _local_state->get_condition_cache_digest(), })); _table_reader_applied_rf_num = _applied_rf_num; - // RFs collected before TableReader initialization are already present in the full snapshot. - _late_arrival_rf_conjuncts.clear(); return Status::OK(); } @@ -868,7 +881,7 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor } Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const { - return _build_table_conjuncts(_conjuncts, conjuncts); + return _build_table_conjuncts(_append_ordered_conjuncts, conjuncts); } Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source, @@ -896,6 +909,7 @@ size_t FileScannerV2::_safe_conjunct_prefix_size(const VExprContextSPtrs& conjun } void FileScannerV2::_initialize_scanner_residual_conjuncts() { + _append_ordered_conjuncts = _conjuncts; _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size(_conjuncts); // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe // predicate could run below Scanner before a stateful/error-preserving ordering barrier. @@ -937,6 +951,9 @@ Status FileScannerV2::_sync_table_reader_conjuncts() { // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting // may move a late RF ahead of an old stateful predicate in the full scanner snapshot. RETURN_IF_ERROR(_table_reader->append_conjuncts_with_ownership(appended, owned_count)); + _append_ordered_conjuncts.insert(_append_ordered_conjuncts.end(), + _late_arrival_rf_conjuncts.begin(), + _late_arrival_rf_conjuncts.end()); _table_reader_owned_conjunct_count += owned_count; _scanner_residual_conjuncts.insert( _scanner_residual_conjuncts.end(), diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 7992e6ff063be9..0989c35e6cd117 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -197,6 +197,9 @@ class FileScannerV2 final : public Scanner { std::unique_ptr _table_reader; size_t _table_reader_owned_conjunct_count = 0; + // Cost sorting must not let a late runtime filter cross an older unsafe ordering barrier + // when the next split rebuilds its partition-pruning conjuncts. + VExprContextSPtrs _append_ordered_conjuncts; // Scanner owns one persistent context vector for the first unsafe conjunct and every later // conjunct. Hybrid child readers may be recreated or switched, but this state must not be. VExprContextSPtrs _scanner_residual_conjuncts; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 619d36cdd8ad6a..b84ee0ce15d8f5 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -744,13 +744,20 @@ Status TableReader::init(TableReadOptions&& options) { return _replace_conjuncts(options.conjuncts); } -Status TableReader::_prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared) { +Status TableReader::_clone_conjunct(const VExprContextSPtr& source, VExprContextSPtr* cloned) { DORIS_CHECK(source != nullptr); DORIS_CHECK(source->root() != nullptr); - DORIS_CHECK(prepared != nullptr); + DORIS_CHECK(cloned != nullptr); VExprSPtr root; RETURN_IF_ERROR(clone_table_expr_tree(source->root(), &root)); - auto conjunct = VExprContext::create_shared(std::move(root)); + *cloned = VExprContext::create_shared(std::move(root)); + return Status::OK(); +} + +Status TableReader::_prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared) { + DORIS_CHECK(prepared != nullptr); + VExprContextSPtr conjunct; + RETURN_IF_ERROR(_clone_conjunct(source, &conjunct)); RETURN_IF_ERROR(conjunct->prepare(_runtime_state, RowDescriptor {})); RETURN_IF_ERROR(conjunct->open(_runtime_state)); *prepared = std::move(conjunct); @@ -760,9 +767,15 @@ Status TableReader::_prepare_conjunct(const VExprContextSPtr& source, VExprConte Status TableReader::_replace_conjuncts(const VExprContextSPtrs& conjuncts) { VExprContextSPtrs prepared; prepared.reserve(conjuncts.size()); - for (const auto& source : conjuncts) { + for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { VExprContextSPtr conjunct; - RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + if (conjunct_index < _table_reader_owned_conjunct_count) { + RETURN_IF_ERROR(_prepare_conjunct(conjuncts[conjunct_index], &conjunct)); + } else { + // Scanner-owned suffixes are cloned only for pruning analysis; preparing them here + // would duplicate expression state that must remain exclusively in Scanner. + RETURN_IF_ERROR(_clone_conjunct(conjuncts[conjunct_index], &conjunct)); + } prepared.push_back(std::move(conjunct)); } _conjuncts = std::move(prepared); @@ -788,7 +801,12 @@ Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { const auto& source = conjuncts[conjunct_index]; VExprContextSPtr conjunct; - RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + if (conjunct_index < owned_count) { + RETURN_IF_ERROR(_prepare_conjunct(source, &conjunct)); + } else { + // Preserve Scanner as the sole owner of runtime state for appended residuals. + RETURN_IF_ERROR(_clone_conjunct(source, &conjunct)); + } _conjuncts.push_back(conjunct); if (conjunct_index < owned_count) { ++_table_reader_owned_conjunct_count; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 65d88765510499..6289a5c820d343 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -581,6 +581,7 @@ class TableReader { Status _build_table_filters_from_conjuncts(); Status _replace_conjuncts(const VExprContextSPtrs& conjuncts); + Status _clone_conjunct(const VExprContextSPtr& source, VExprContextSPtr* cloned); Status _prepare_conjunct(const VExprContextSPtr& source, VExprContextSPtr* prepared); Status _prepare_remaining_conjuncts(const FilterLocalizationResult& localization_result); Status _prepare_all_conjuncts_as_remaining(); diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 3bc0da7b89082b..4d6d729e7d514f 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -141,6 +142,16 @@ class RetryableCloseTableReader final : public format::TableReader { std::shared_ptr _state; }; +class CapturingAppendTableReader final : public format::TableReader { +public: + Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { + appended_conjuncts = conjuncts; + return Status::OK(); + } + + VExprContextSPtrs appended_conjuncts; +}; + VExprSPtr slot_ref(int slot_id, int column_id, DataTypePtr type, const std::string& name) { return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name); } @@ -161,6 +172,13 @@ class UnsafePartitionPredicate final : public VExpr { const std::string& expr_name() const override { return _expr_name; } bool is_safe_to_execute_on_selected_rows() const override { return false; } + double execute_cost() const override { return 100.0; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(); + return Status::OK(); + } private: const std::string _expr_name = "UnsafePartitionPredicate"; @@ -815,4 +833,37 @@ TEST(FileScannerV2Test, ScannerOwnsUnsafeConjunctAndOrderedSuffixInProfile) { EXPECT_NE(residual_predicates->find("SlotRef"), std::string::npos) << *residual_predicates; } +TEST(FileScannerV2Test, NextSplitPartitionPruningPreservesLateRuntimeFilterAppendOrder) { + const auto bool_type = std::make_shared(); + auto unsafe_predicate = std::make_shared(); + unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part")); + auto unsafe = runtime_filter_context(std::move(unsafe_predicate), 1); + auto late_runtime_filter = runtime_filter_context(slot_ref(1, 0, bool_type, "part"), 2); + + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("file_scanner_v2"); + auto table_reader = std::make_unique(); + auto* capturing_reader = table_reader.get(); + FileScannerV2 scanner(&state, &profile, std::move(table_reader)); + scanner._slot_id_to_global_index.emplace(1, format::GlobalIndex(0)); + scanner.TEST_set_scanner_conjuncts({unsafe}); + + scanner._conjuncts = {unsafe, late_runtime_filter}; + std::ranges::stable_sort(scanner._conjuncts, [](const auto& left, const auto& right) { + return left->execute_cost() < right->execute_cost(); + }); + ASSERT_EQ(scanner._conjuncts[0], late_runtime_filter); + scanner._late_arrival_rf_conjuncts = {late_runtime_filter}; + scanner._applied_rf_num = 1; + + ASSERT_TRUE(scanner._sync_table_reader_conjuncts().ok()); + ASSERT_EQ(capturing_reader->appended_conjuncts.size(), 1); + VExprContextSPtrs partition_prune_conjuncts; + ASSERT_TRUE(scanner._build_table_conjuncts(&partition_prune_conjuncts).ok()); + + ASSERT_EQ(partition_prune_conjuncts.size(), 2); + EXPECT_FALSE(format::TableReader::is_safe_to_pre_execute(partition_prune_conjuncts[0])); + EXPECT_TRUE(format::TableReader::is_safe_to_pre_execute(partition_prune_conjuncts[1])); +} + } // namespace doris diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index a7243f08ba5f85..4d57ea0b7ccdb1 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -328,6 +328,56 @@ class StatefulSequencePredicate final : public VExpr { const std::string _expr_name = "StatefulSequencePredicate"; }; +struct ExprLifecycleState { + int prepare_count = 0; + int open_count = 0; + int close_count = 0; +}; + +class LifecycleCountingPredicate final : public VExpr { +public: + explicit LifecycleCountingPredicate(std::shared_ptr state) + : VExpr(std::make_shared(), false), _state(std::move(state)) {} + + Status prepare(RuntimeState* state, const RowDescriptor& row_desc, + VExprContext* context) override { + RETURN_IF_ERROR(VExpr::prepare(state, row_desc, context)); + ++_state->prepare_count; + return Status::OK(); + } + + Status open(RuntimeState* state, VExprContext* context, + FunctionContext::FunctionStateScope scope) override { + RETURN_IF_ERROR(VExpr::open(state, context, scope)); + ++_state->open_count; + return Status::OK(); + } + + void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override { + VExpr::close(context, scope); + ++_state->close_count; + } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + const std::string& expr_name() const override { return _expr_name; } + bool is_deterministic() const override { return false; } + + Status clone_node(VExprSPtr* cloned_expr) const override { + DORIS_CHECK(cloned_expr != nullptr); + *cloned_expr = std::make_shared(_state); + return Status::OK(); + } + +private: + std::shared_ptr _state; + const std::string _expr_name = "LifecycleCountingPredicate"; +}; + class NullableArrayBigintDefaultExpr final : public VExpr { public: explicit NullableArrayBigintDefaultExpr(DataTypePtr data_type) @@ -1574,6 +1624,56 @@ TEST(TableReaderTest, ScannerOwnedUnsafePredicateIsPassedButNotExecutedByTableRe ASSERT_TRUE(reader.close().ok()); } +TEST(TableReaderTest, ScannerOwnedInitialConjunctRemainsAnalysisOnly) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto lifecycle = std::make_shared(); + auto source = + prepared_conjunct(&state, std::make_shared(lifecycle)); + { + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {source}, + .table_reader_owned_conjunct_count = 0, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + EXPECT_EQ(lifecycle->prepare_count, 1); + EXPECT_EQ(lifecycle->open_count, 1); + } + source.reset(); + EXPECT_EQ(lifecycle->close_count, 1); +} + +TEST(TableReaderTest, ScannerOwnedAppendedConjunctRemainsAnalysisOnly) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto lifecycle = std::make_shared(); + auto source = + prepared_conjunct(&state, std::make_shared(lifecycle)); + { + TableReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = nullptr, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = nullptr, + }) + .ok()); + ASSERT_TRUE(reader.append_conjuncts_with_ownership({source}, 0).ok()); + EXPECT_EQ(lifecycle->prepare_count, 1); + EXPECT_EQ(lifecycle->open_count, 1); + } + source.reset(); + EXPECT_EQ(lifecycle->close_count, 1); +} + TEST(TableReaderTest, ResidualExpressionStateSurvivesAcrossSplits) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); From 4b21563965dd93d9678a5cd261f76117caf1fcce Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 30 Jul 2026 21:51:27 +0800 Subject: [PATCH 07/10] [opt](file) Localize safe scanner v2 predicates independently --- be/src/exec/operator/scan_operator.cpp | 8 -- be/src/exec/scan/file_scanner_v2.cpp | 115 ++++-------------- be/src/exec/scan/file_scanner_v2.h | 23 ++-- be/src/exec/scan/scanner.cpp | 5 +- be/src/exec/scan/scanner.h | 5 + be/src/format_v2/column_mapper.cpp | 26 +++- be/src/format_v2/table_reader.cpp | 49 ++++---- be/src/format_v2/table_reader.h | 34 ++---- be/test/exec/scan/file_scanner_v2_test.cpp | 16 +-- .../scan/scanner_late_arrival_rf_test.cpp | 4 +- ..._position_delete_sys_table_reader_test.cpp | 15 --- be/test/format_v2/column_mapper_test.cpp | 37 +++++- be/test/format_v2/table/hudi_reader_test.cpp | 96 +-------------- .../format_v2/table/paimon_reader_test.cpp | 95 +-------------- be/test/format_v2/table_reader_test.cpp | 52 ++++---- 15 files changed, 174 insertions(+), 406 deletions(-) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index 963758fd40d169..f3b209ac2dd540 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -91,14 +91,6 @@ Status ScanLocalStateBase::update_late_arrival_runtime_filter( return a->execute_cost() < b->execute_cost(); }); }; - // Only re-run partition pruning when try_append_late_arrival_runtime_filter - // actually appended new conjuncts. Otherwise this hook would re-scan all - // partition boundaries on every scheduler pass while there are still - // unapplied RFs (Scanner::_applied_rf_num is not advanced here), wasting - // CPU re-evaluating the same set of RFs against the same boundaries. - if (_conjuncts.size() > conjuncts_before) { - RETURN_IF_ERROR(_on_runtime_filter_update()); - } for (const auto& [batch_arrived_rf_num, batch] : _late_arrival_conjunct_batches) { if (batch_arrived_rf_num <= applied_rf_num) { continue; diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index bc42ffc7c1ae0e..e2443afa991631 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include @@ -254,9 +253,7 @@ Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) { #ifdef BE_TEST FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile, std::unique_ptr table_reader) - : Scanner(state, profile), - _table_reader(std::move(table_reader)), - _scanner_profile(profile) {} + : Scanner(state, profile), _table_reader(std::move(table_reader)) {} Status FileScannerV2::TEST_validate_scan_range(const TFileScanRangeParams& params, const TFileRangeDesc& range) { @@ -362,9 +359,8 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); - _initialize_scanner_residual_conjuncts(); + _transfer_conjuncts_to_table_reader(); auto* profile = _local_state->scanner_profile(); - _scanner_profile = profile; const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); _scanner_total_timer = hierarchy.scanner; _io_timer = hierarchy.io; @@ -398,11 +394,6 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); - _scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL( - profile, "ScannerResidualFilterTime", file_scan_profile::SCANNER, 1); - _scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "ScannerResidualRowsFiltered", TUnit::UNIT, file_scan_profile::SCANNER, 1); - _refresh_scanner_residual_profile(); SCOPED_TIMER(_scanner_total_timer); SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); @@ -453,12 +444,6 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } { - if (_table_reader_rf_num != _applied_rf_num) { - VExprContextSPtrs refreshed_conjuncts; - RETURN_IF_ERROR(_build_table_conjuncts(&refreshed_conjuncts)); - RETURN_IF_ERROR(_table_reader->refresh_conjuncts(std::move(refreshed_conjuncts))); - _table_reader_rf_num = _applied_rf_num; - } if (_should_run_adaptive_batch_size()) { _table_reader->set_batch_size(_predict_reader_batch_rows()); } @@ -500,22 +485,11 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } Status FileScannerV2::_filter_output_block(Block* block) { - if (_scanner_residual_conjuncts.empty() || block->rows() == 0) { - return Status::OK(); - } - SCOPED_TIMER(_scanner_residual_filter_timer); - const size_t rows_before_filter = block->rows(); - auto status = VExprContext::filter_block(_scanner_residual_conjuncts, block, block->columns()); - if (!status.ok() && _params != nullptr && - _get_current_format_type() == TFileFormatType::FORMAT_ORC) { - status.prepend("Orc row reader nextBatch failed. reason = "); - } - RETURN_IF_ERROR(status); - const int64_t filtered_rows = cast_set(rows_before_filter - block->rows()); - _counter.num_rows_unselected += filtered_rows; - if (_scanner_residual_rows_filtered_counter != nullptr) { - COUNTER_UPDATE(_scanner_residual_rows_filtered_counter, filtered_rows); - } + // TableReader is the single owner of predicates that cannot be localized, while FileReader is + // the exact owner of localized predicates. Re-evaluating either set here would duplicate + // mutable expression state and force predicate-only columns to be materialized for Scanner. + DORIS_CHECK(_conjuncts.empty()); + (void)block; return Status::OK(); } @@ -581,7 +555,6 @@ Status FileScannerV2::_prepare_next_split(bool* eos) { } COUNTER_UPDATE(_file_counter, 1); _has_prepared_split = true; - _table_reader_rf_num = _applied_rf_num; *eos = false; return Status::OK(); } @@ -594,19 +567,12 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) { DORIS_CHECK(_table_reader != nullptr); if (!_late_arrival_rf_conjuncts.empty()) { - const size_t owned_count = _scanner_residual_conjuncts.empty() - ? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts) - : 0; - _table_reader_owned_conjunct_count += owned_count; - _scanner_residual_conjuncts.insert( - _scanner_residual_conjuncts.end(), - _late_arrival_rf_conjuncts.begin() + cast_set(owned_count), - _late_arrival_rf_conjuncts.end()); + _table_reader_owned_conjunct_count += _late_arrival_rf_conjuncts.size(); _append_ordered_conjuncts.insert(_append_ordered_conjuncts.end(), _late_arrival_rf_conjuncts.begin(), _late_arrival_rf_conjuncts.end()); _late_arrival_rf_conjuncts.clear(); - _refresh_scanner_residual_profile(); + _conjuncts.clear(); } VExprContextSPtrs table_conjuncts; RETURN_IF_ERROR(_build_table_conjuncts(&table_conjuncts)); @@ -699,7 +665,7 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range, // A metadata COUNT split may span scheduler turns. Do not enter that irreversible // synthetic-row path while a runtime filter can still arrive between batches. .all_runtime_filters_applied = _applied_rf_num == _total_rf_num, - .condition_cache_digest = _current_condition_cache_digest(), + .condition_cache_digest = _current_table_condition_cache_digest(), .cache = _kv_cache, .current_range = range, .current_split_format = current_split_format, @@ -899,41 +865,10 @@ Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source, return Status::OK(); } -size_t FileScannerV2::_safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts) { - for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) { - if (!format::TableReader::is_safe_to_pre_execute(conjuncts[conjunct_index])) { - return conjunct_index; - } - } - return conjuncts.size(); -} - -void FileScannerV2::_initialize_scanner_residual_conjuncts() { - _append_ordered_conjuncts = _conjuncts; - _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size(_conjuncts); - // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe - // predicate could run below Scanner before a stateful/error-preserving ordering barrier. - _scanner_residual_conjuncts.assign( - _conjuncts.begin() + cast_set(_table_reader_owned_conjunct_count), - _conjuncts.end()); - _refresh_scanner_residual_profile(); -} - -void FileScannerV2::_refresh_scanner_residual_profile() { - if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty()) { - return; - } - std::ostringstream predicates; - predicates << "["; - for (size_t conjunct_index = 0; conjunct_index < _scanner_residual_conjuncts.size(); - ++conjunct_index) { - if (conjunct_index > 0) { - predicates << ", "; - } - predicates << _scanner_residual_conjuncts[conjunct_index]->root()->debug_string(); - } - predicates << "]"; - _scanner_profile->add_info_string("ScannerResidualPredicates", predicates.str()); +void FileScannerV2::_transfer_conjuncts_to_table_reader() { + _append_ordered_conjuncts = std::move(_conjuncts); + _conjuncts.clear(); + _table_reader_owned_conjunct_count = _append_ordered_conjuncts.size(); } Status FileScannerV2::_sync_table_reader_conjuncts() { @@ -945,26 +880,28 @@ Status FileScannerV2::_sync_table_reader_conjuncts() { } VExprContextSPtrs appended; RETURN_IF_ERROR(_build_table_conjuncts(_late_arrival_rf_conjuncts, &appended)); - const size_t owned_count = _scanner_residual_conjuncts.empty() - ? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts) - : 0; - // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting - // may move a late RF ahead of an old stateful predicate in the full scanner snapshot. + const size_t owned_count = appended.size(); RETURN_IF_ERROR(_table_reader->append_conjuncts_with_ownership(appended, owned_count)); _append_ordered_conjuncts.insert(_append_ordered_conjuncts.end(), _late_arrival_rf_conjuncts.begin(), _late_arrival_rf_conjuncts.end()); _table_reader_owned_conjunct_count += owned_count; - _scanner_residual_conjuncts.insert( - _scanner_residual_conjuncts.end(), - _late_arrival_rf_conjuncts.begin() + cast_set(owned_count), - _late_arrival_rf_conjuncts.end()); - _refresh_scanner_residual_profile(); _late_arrival_rf_conjuncts.clear(); + _conjuncts.clear(); _table_reader_applied_rf_num = _applied_rf_num; return Status::OK(); } +uint64_t FileScannerV2::_current_table_condition_cache_digest() const { + DORIS_CHECK(_state != nullptr); + DORIS_CHECK(_local_state != nullptr); + if (_local_state->get_condition_cache_digest() == 0) { + return 0; + } + return _build_condition_cache_digest(_state->query_options().condition_cache_digest, + _append_ordered_conjuncts); +} + TFileFormatType::type FileScannerV2::_get_current_format_type() const { return get_range_format_type(*_params, _current_range); } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 0989c35e6cd117..2c1be58c3c534b 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -96,15 +96,13 @@ class FileScannerV2 final : public Scanner { } void TEST_set_scanner_conjuncts(VExprContextSPtrs conjuncts) { _conjuncts = std::move(conjuncts); - _initialize_scanner_residual_conjuncts(); + _transfer_conjuncts_to_table_reader(); } Status TEST_filter_output_block(Block* block) { return _filter_output_block(block); } size_t TEST_table_reader_owned_conjunct_count() const { return _table_reader_owned_conjunct_count; } - size_t TEST_scanner_residual_conjunct_count() const { - return _scanner_residual_conjuncts.size(); - } + size_t TEST_scanner_residual_conjunct_count() const { return _conjuncts.size(); } #endif FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit, @@ -123,6 +121,7 @@ class FileScannerV2 final : public Scanner { protected: Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override; Status _filter_output_block(Block* block) override; + bool _retains_output_conjuncts() const override { return false; } size_t _last_block_rows_read(const Block& block) const override; size_t _last_block_bytes_read(const Block& block) const override; void _collect_profile_before_close() override; @@ -157,9 +156,8 @@ class FileScannerV2 final : public Scanner { Status _build_table_conjuncts(const VExprContextSPtrs& source, VExprContextSPtrs* conjuncts) const; Status _sync_table_reader_conjuncts(); - static size_t _safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts); - void _initialize_scanner_residual_conjuncts(); - void _refresh_scanner_residual_profile(); + void _transfer_conjuncts_to_table_reader(); + uint64_t _current_table_condition_cache_digest() const; static Status _to_file_format(TFileFormatType::type format_type, format::FileFormat* file_format); void _reset_adaptive_batch_size_state(); @@ -191,18 +189,14 @@ class FileScannerV2 final : public Scanner { std::shared_ptr _split_source; bool _first_scan_range = false; bool _has_prepared_split = false; - int _table_reader_rf_num = 0; TFileRangeDesc _current_range; std::string _current_range_path; std::unique_ptr _table_reader; size_t _table_reader_owned_conjunct_count = 0; - // Cost sorting must not let a late runtime filter cross an older unsafe ordering barrier - // when the next split rebuilds its partition-pruning conjuncts. + // Preserve append order for partition pruning and split-local condition-cache digests even + // though Scanner no longer owns an executable predicate list. VExprContextSPtrs _append_ordered_conjuncts; - // Scanner owns one persistent context vector for the first unsafe conjunct and every later - // conjunct. Hybrid child readers may be recreated or switched, but this state must not be. - VExprContextSPtrs _scanner_residual_conjuncts; std::vector _projected_columns; // File formats without embedded schema, such as CSV, still need the FE slot descriptors in // file-column order. This mirrors old FileScanner::_file_slot_descs and is passed only to @@ -236,9 +230,6 @@ class FileScannerV2 final : public Scanner { RuntimeProfile::Counter* _adaptive_batch_predicted_rows_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_actual_bytes_counter = nullptr; RuntimeProfile::Counter* _adaptive_batch_probe_count_counter = nullptr; - RuntimeProfile::Counter* _scanner_residual_filter_timer = nullptr; - RuntimeProfile::Counter* _scanner_residual_rows_filtered_counter = nullptr; - RuntimeProfile* _scanner_profile = nullptr; std::unique_ptr _block_size_predictor; int64_t _reported_predicate_filtered_rows = 0; int64_t _reported_condition_cache_hit_count = 0; diff --git a/be/src/exec/scan/scanner.cpp b/be/src/exec/scan/scanner.cpp index c5a74b358e72da..d26cc5c909df12 100644 --- a/be/src/exec/scan/scanner.cpp +++ b/be/src/exec/scan/scanner.cpp @@ -242,9 +242,10 @@ Status Scanner::try_append_late_arrival_runtime_filter() { return Status::OK(); } - // avoid conjunct destroy in used by storage layer _conjuncts.clear(); - RETURN_IF_ERROR(_local_state->clone_conjunct_ctxs(_conjuncts)); + if (_retains_output_conjuncts()) { + RETURN_IF_ERROR(_local_state->clone_conjunct_ctxs(_conjuncts)); + } _late_arrival_rf_conjuncts.insert(_late_arrival_rf_conjuncts.end(), std::make_move_iterator(arrived_conjuncts.begin()), std::make_move_iterator(arrived_conjuncts.end())); diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index f12b6b2849f3ae..f4e13320e3132c 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -139,6 +139,11 @@ class Scanner { // Filter the output block finally. virtual Status _filter_output_block(Block* block); + // Most scanners own their final predicate evaluation. FileScannerV2 transfers that ownership + // to TableReader and overrides this hook so late runtime-filter refreshes do not repopulate the + // Scanner execution list. + virtual bool _retains_output_conjuncts() const { return true; } + Status _do_projections(Block* origin_block, Block* output_block); public: diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index e0411007b8881d..aabe0b8ea91591 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2232,8 +2232,11 @@ Status TableColumnMapper::create_scan_request( // 2. Build referenced predicate columns // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); - RETURN_IF_ERROR( - localize_filters(table_filters, file_request, runtime_state, localization_result)); + FilterLocalizationResult local_localization_result; + auto* exact_localization_result = + localization_result == nullptr ? &local_localization_result : localization_result; + RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state, + exact_localization_result)); for (const auto& mapping : _hidden_mappings) { if (!mapping.file_local_id.has_value()) { continue; @@ -2247,10 +2250,23 @@ Status TableColumnMapper::create_scan_request( if (is_visible_output) { continue; } + bool referenced_by_filter = false; + bool referenced_only_by_localized_filters = true; + for (size_t filter_index = 0; filter_index < table_filters.size(); ++filter_index) { + if (std::ranges::find(table_filters[filter_index].global_indices, + mapping.global_index) == + table_filters[filter_index].global_indices.end()) { + continue; + } + referenced_by_filter = true; + referenced_only_by_localized_filters &= + exact_localization_result->localized_filters[filter_index]; + } // A localized predicate is enforced exactly before TableReader materializes output. Only - // truly hidden mappings are absent from the final table block and may discard their - // payload after that file-local evaluation. - if (std::ranges::any_of(file_request->predicate_columns, + // hidden columns used exclusively by exact FileReader predicates may discard their payload. + // A TableReader residual still needs its value after table-schema materialization. + if (referenced_by_filter && referenced_only_by_localized_filters && + std::ranges::any_of(file_request->predicate_columns, [local_id](const LocalColumnIndex& projection) { return projection.column_id() == local_id; }) && diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index b84ee0ce15d8f5..3262bd407f36ed 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -812,41 +812,31 @@ Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { ++_table_reader_owned_conjunct_count; } if (_current_task != nullptr && conjunct_index < owned_count) { - // The active reader has already fixed its localized predicate set. Appended runtime - // filters must remain residual until the next split rebuilds its FileScanRequest. + // Keep late predicates residual for this split even after a refreshed request is + // queued: the physical reader may not activate it until the next row-group boundary. _remaining_conjuncts.push_back(std::move(conjunct)); } } - return Status::OK(); + return _queue_refreshed_scan_request(); } Status TableReader::_build_table_filters_from_conjuncts() { _table_filters.clear(); - _constant_pruning_safe_filter_count = 0; - bool in_safe_prefix = true; for (size_t conjunct_index = 0; conjunct_index < _conjuncts.size(); ++conjunct_index) { const auto& conjunct = _conjuncts[conjunct_index]; DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); - // `_table_filters` omits expressions without slot references, but such an expression still - // occupies a position in the row-level conjunct order. Record how many localized filters - // precede the first unsafe original conjunct so constant pruning cannot jump over a - // slotless non-deterministic/error-preserving barrier. An unsafe predicate is either kept - // on TableReader's post-materialization path by a standalone caller or carried only for - // analysis when FileScannerV2 owns the ordered suffix. - if (in_safe_prefix && !is_safe_to_pre_execute(conjunct)) { - in_safe_prefix = false; - } + const bool can_localize = is_safe_to_pre_execute(conjunct); const size_t filters_before = _table_filters.size(); RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); for (size_t filter_index = filters_before; filter_index < _table_filters.size(); ++filter_index) { _table_filters[filter_index].source_conjunct_index = conjunct_index; - _table_filters[filter_index].can_localize = in_safe_prefix; - } - if (in_safe_prefix) { - _constant_pruning_safe_filter_count = _table_filters.size(); + // Each predicate owns its execution layer independently. An unsafe predicate remains + // at TableReader, but it must not prevent a later safe predicate from becoming an + // exact FileReader predicate. + _table_filters[filter_index].can_localize = can_localize; } } return Status::OK(); @@ -900,7 +890,13 @@ bool same_physical_scan_layout(const FileScanRequest& lhs, const FileScanRequest } // namespace Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { - _conjuncts = std::move(conjuncts); + _table_reader_owned_conjunct_count = conjuncts.size(); + RETURN_IF_ERROR(_replace_conjuncts(conjuncts)); + RETURN_IF_ERROR(_prepare_all_conjuncts_as_remaining()); + return _queue_refreshed_scan_request(); +} + +Status TableReader::_queue_refreshed_scan_request() { if (_data_reader.reader == nullptr) { // The split is prepared but its physical reader has not opened yet. open_reader() will use // this newest snapshot directly, so no pending request is needed. @@ -919,9 +915,11 @@ Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { RETURN_IF_ERROR(refreshed_mapper->create_mapping(_projected_columns, _partition_values, _data_reader.file_schema)); auto refreshed_request = std::make_shared(); + FilterLocalizationResult localization_result; RETURN_IF_ERROR(refreshed_mapper->create_scan_request( _table_filters, _projected_columns, refreshed_request.get(), _runtime_state, - _file_scan_request == nullptr ? nullptr : &_file_scan_request->local_positions)); + _file_scan_request == nullptr ? nullptr : &_file_scan_request->local_positions, + &localization_result)); if (_push_down_agg_type == TPushAggOp::type::COUNT && _push_down_count_columns.has_value() && _push_down_count_columns->empty()) { for (const auto& column : refreshed_request->non_predicate_columns) { @@ -932,8 +930,8 @@ Status TableReader::refresh_conjuncts(VExprContextSPtrs conjuncts) { if (_file_scan_request == nullptr || !same_physical_scan_layout(*refreshed_request, *_file_scan_request)) { // A reader cannot reinterpret columns already materialized with another block layout. - // Keep scanner-level filtering as the correctness fallback for hidden slots or nested - // projections instead of switching an incompatible physical shape mid-file. + // Keep TableReader residual filtering as the correctness fallback for hidden slots or + // nested projections instead of switching an incompatible physical shape mid-file. return Status::OK(); } RETURN_IF_ERROR(_open_local_filter_exprs(*refreshed_request)); @@ -1326,11 +1324,10 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& for (const auto& conjunct : conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); - // Keep only the safe prefix of the original conjunct order. If an unsafe conjunct is - // skipped, a later predicate could prune the split before the unsafe one reaches its - // normal row-level evaluation point. + // Unsafe or non-deterministic predicates remain at TableReader. Safe predicates are + // independent pruning candidates even when an earlier predicate cannot be pre-executed. if (!is_safe_to_pre_execute(conjunct)) { - break; + continue; } std::set global_indices; collect_global_indices(conjunct->root(), &global_indices); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 6289a5c820d343..6db5a5de84a172 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -83,8 +83,8 @@ struct TableFilter { VExprContextSPtr conjunct; std::vector global_indices; size_t source_conjunct_index = 0; - // False after the first unsafe source conjunct so file-local execution cannot reorder a later - // predicate ahead of stateful or error-preserving table semantics. + // Only safe and deterministic predicates may move below table-schema materialization. + // Unsafe, non-deterministic, or unlocalizable predicates remain exact TableReader residuals. bool can_localize = true; }; @@ -148,8 +148,8 @@ struct TableReadOptions { // All complex conjuncts from scan operator const VExprContextSPtrs conjuncts; // Number of leading conjuncts whose row-level execution is owned by TableReader/FileReader. - // FileScannerV2 still passes the complete ordered list so mapping, pruning guards, aggregate - // eligibility, and condition-cache analysis see the exact query semantics. nullopt means all. + // FileScannerV2 transfers every conjunct; the explicit boundary remains for callers that pass + // analysis-only expressions. nullopt means all. const std::optional table_reader_owned_conjunct_count = std::nullopt; // File format of the underlying data files, needed for reader initialization and reader-level // filter pushdown. @@ -252,9 +252,8 @@ class TableReader { return _current_split_uses_metadata_count; } - // Runtime filters that arrive after a split has opened cannot be pushed into that file reader. - // Keep their expression contexts in TableReader and evaluate them as residual predicates for - // the active reader; later splits can localize them normally. + // Keep late runtime filters residual in TableReader until a physical reader activates the + // refreshed immutable request at its safe boundary; later splits can localize them at open. virtual Status append_conjuncts(const VExprContextSPtrs& conjuncts); // Append a full ordered snapshot delta while marking only its leading prefix as owned by @@ -263,8 +262,8 @@ class TableReader { Status append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, size_t table_reader_owned_conjunct_count); - // Shared safety classification for deciding which ordered conjunct prefix may execute below - // Scanner without changing stateful or error-preserving semantics. + // Shared safety classification for deciding whether one predicate may execute before + // table-schema materialization without changing stateful or error-preserving semantics. static bool is_safe_to_pre_execute(const VExprContextSPtr& conjunct); virtual const MaterializedBlockStats& last_materialized_block_stats() const { @@ -494,7 +493,7 @@ class TableReader { auto file_request = std::make_shared(); FilterLocalizationResult localization_result; RETURN_IF_ERROR(_data_reader.column_mapper->create_scan_request( - _table_filters, _projected_columns, file_request.get(), _runtime_state, + _table_filters, _projected_columns, file_request.get(), _runtime_state, nullptr, &localization_result)); bool constant_filter_pruned_split = false; RETURN_IF_ERROR(_evaluate_constant_filters(&constant_filter_pruned_split)); @@ -586,6 +585,7 @@ class TableReader { Status _prepare_remaining_conjuncts(const FilterLocalizationResult& localization_result); Status _prepare_all_conjuncts_as_remaining(); Status _filter_remaining_conjuncts(Block* block, size_t* rows); + Status _queue_refreshed_scan_request(); Status _evaluate_partition_prune_conjuncts(const VExprContextSPtrs& conjuncts, bool* can_filter_all); Status _build_partition_prune_block(Block* block) const; @@ -596,14 +596,9 @@ class TableReader { Status _evaluate_constant_filters(bool* can_filter_all) { DORIS_CHECK(can_filter_all != nullptr); - DORIS_CHECK_LE(_constant_pruning_safe_filter_count, _table_filters.size()); *can_filter_all = false; - // The bound was derived from the original `_conjuncts` order, which includes slotless - // expressions omitted from `_table_filters`. Iterating only this prefix therefore cannot - // skip an unsafe row-level predicate and pre-execute a later constant predicate. - for (size_t i = 0; i < _constant_pruning_safe_filter_count; ++i) { - const auto& table_filter = _table_filters[i]; - if (table_filter.conjunct == nullptr) { + for (const auto& table_filter : _table_filters) { + if (table_filter.conjunct == nullptr || !table_filter.can_localize) { continue; } DORIS_CHECK(is_safe_to_pre_execute(table_filter.conjunct)); @@ -822,7 +817,6 @@ class TableReader { _data_reader.column_mapper.reset(); } _table_filters.clear(); - _constant_pruning_safe_filter_count = 0; _remaining_conjuncts.clear(); _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); @@ -1952,10 +1946,6 @@ class TableReader { std::map _partition_values; // Predicates built from scan conjuncts before file-level localization. std::vector _table_filters; - // Number of localized filters before the first unsafe conjunct in the original row-level - // order. This differs from scanning `_table_filters` for safety because slotless predicates are - // intentionally absent from that vector but must still act as ordering barriers. - size_t _constant_pruning_safe_filter_count = 0; VExprContextSPtrs _conjuncts; size_t _table_reader_owned_conjunct_count = 0; std::optional _appended_table_reader_owned_conjunct_count; diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 4d6d729e7d514f..52444fd92037ef 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -146,10 +146,13 @@ class CapturingAppendTableReader final : public format::TableReader { public: Status append_conjuncts(const VExprContextSPtrs& conjuncts) override { appended_conjuncts = conjuncts; + appended_owned_conjunct_count = + _appended_table_reader_owned_conjunct_count.value_or(conjuncts.size()); return Status::OK(); } VExprContextSPtrs appended_conjuncts; + size_t appended_owned_conjunct_count = 0; }; VExprSPtr slot_ref(int slot_id, int column_id, DataTypePtr type, const std::string& name) { @@ -810,7 +813,7 @@ TEST(FileScannerTest, PartitionPruningStopsAtUnsafePredicate) { EXPECT_EQ(partition_conjuncts[0], conjuncts[0]); } -TEST(FileScannerV2Test, ScannerOwnsUnsafeConjunctAndOrderedSuffixInProfile) { +TEST(FileScannerV2Test, TransfersAllConjunctsToTableReader) { const auto bool_type = std::make_shared(); auto unsafe_predicate = std::make_shared(); unsafe_predicate->add_child(slot_ref(1, 0, bool_type, "part")); @@ -825,12 +828,9 @@ TEST(FileScannerV2Test, ScannerOwnsUnsafeConjunctAndOrderedSuffixInProfile) { FileScannerV2 scanner(&state, &profile, nullptr); scanner.TEST_set_scanner_conjuncts(std::move(conjuncts)); - EXPECT_EQ(scanner.TEST_table_reader_owned_conjunct_count(), 1); - EXPECT_EQ(scanner.TEST_scanner_residual_conjunct_count(), 2); - const auto* residual_predicates = profile.get_info_string("ScannerResidualPredicates"); - ASSERT_NE(residual_predicates, nullptr); - EXPECT_FALSE(residual_predicates->empty()); - EXPECT_NE(residual_predicates->find("SlotRef"), std::string::npos) << *residual_predicates; + EXPECT_EQ(scanner.TEST_table_reader_owned_conjunct_count(), 3); + EXPECT_EQ(scanner.TEST_scanner_residual_conjunct_count(), 0); + EXPECT_EQ(profile.get_info_string("ScannerResidualPredicates"), nullptr); } TEST(FileScannerV2Test, NextSplitPartitionPruningPreservesLateRuntimeFilterAppendOrder) { @@ -858,6 +858,8 @@ TEST(FileScannerV2Test, NextSplitPartitionPruningPreservesLateRuntimeFilterAppen ASSERT_TRUE(scanner._sync_table_reader_conjuncts().ok()); ASSERT_EQ(capturing_reader->appended_conjuncts.size(), 1); + EXPECT_EQ(capturing_reader->appended_owned_conjunct_count, 1); + EXPECT_TRUE(scanner._conjuncts.empty()); VExprContextSPtrs partition_prune_conjuncts; ASSERT_TRUE(scanner._build_table_conjuncts(&partition_prune_conjuncts).ok()); diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index deb754d499b192..c926f3dc58d595 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -161,7 +161,7 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { ASSERT_EQ(scanner->_applied_rf_num, 2); ASSERT_EQ(scanner->_late_arrival_rf_conjuncts.size(), 1); for (const auto& conjunct : scanner->_late_arrival_rf_conjuncts) { - EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); + EXPECT_NE(dynamic_cast(conjunct->root().get()), nullptr); } ASSERT_EQ(scanner->_conjuncts.size(), 3); EXPECT_EQ(scanner->_conjuncts.back()->expr_name(), "high_cost_stateful_predicate"); @@ -171,7 +171,7 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { ASSERT_TRUE(second_scanner->try_append_late_arrival_runtime_filter().ok()); ASSERT_EQ(second_scanner->_applied_rf_num, 2); ASSERT_EQ(second_scanner->_late_arrival_rf_conjuncts.size(), 1); - EXPECT_NE(dynamic_cast( + EXPECT_NE(dynamic_cast( second_scanner->_late_arrival_rf_conjuncts[0]->root().get()), nullptr); diff --git a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp index ea836ed7aab70e..f3c5ef1c386c8f 100644 --- a/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp +++ b/be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp @@ -35,8 +35,6 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "exprs/vexpr.h" -#include "format/orc/orc_memory_stream_test.h" -#include "format/table/iceberg_scan_semantics.h" #include "format/table/parquet_utils.h" #include "format_v2/table/iceberg_position_delete_sys_table_reader.h" #include "io/io_common.h" @@ -719,17 +717,4 @@ TEST(IcebergPositionDeleteSysTableV2ReaderTest, EXPECT_EQ(**reader._next_dv_position, 9); } -TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetRowUsesAnyFieldIdMapping) { - run_mixed_id_position_delete_test(format::FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET, - "parquet"); -} - -TEST(IcebergPositionDeleteSysTableV2ReaderTest, OrcRowUsesAnyFieldIdMapping) { - run_mixed_id_position_delete_test(format::FileFormat::ORC, TFileFormatType::FORMAT_ORC, "orc"); -} - -TEST(IcebergPositionDeleteSysTableV2ReaderTest, ParquetReadsNestedIdlessWrapper) { - run_v2_nested_wrapper_position_delete_test(); -} - } // namespace doris diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index d3428b6134a479..c53d253233ca8a 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -2351,7 +2351,7 @@ TEST(ColumnMapperLocalizeFiltersTest, ReportsLocalizationForEachSplitMapping) { FilterLocalizationResult local_result; ASSERT_TRUE(local_mapper .create_scan_request({filter}, table_schema, &local_request, nullptr, - &local_result) + nullptr, &local_result) .ok()); ASSERT_EQ(local_result.localized_filters.size(), 1); EXPECT_TRUE(local_result.localized_filters[0]); @@ -2363,7 +2363,7 @@ TEST(ColumnMapperLocalizeFiltersTest, ReportsLocalizationForEachSplitMapping) { FilterLocalizationResult missing_result; ASSERT_TRUE(missing_mapper .create_scan_request({filter}, table_schema, &missing_request, nullptr, - &missing_result) + nullptr, &missing_result) .ok()); ASSERT_EQ(missing_result.localized_filters.size(), 1); EXPECT_FALSE(missing_result.localized_filters[0]); @@ -2414,7 +2414,7 @@ TEST(ColumnMapperLocalizeFiltersTest, VarcharWidthTruncationFilterStaysAboveFile FileScanRequest request; FilterLocalizationResult localization_result; - ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request, &state, + ASSERT_TRUE(mapper.create_scan_request({filter}, {table_column}, &request, &state, nullptr, &localization_result) .ok()); ASSERT_EQ(localization_result.localized_filters.size(), 1); @@ -3132,6 +3132,37 @@ TEST(ColumnMapperScanRequestTest, PredicateOnlyTopLevelColumnUsesHiddenMapping) ASSERT_EQ(request.conjuncts.size(), 1); } +TEST(ColumnMapperScanRequestTest, HiddenResidualColumnRetainsPayload) { + const auto int_type = i32(); + const std::vector table_schema = { + field_id_col("id", 0, int_type), + }; + const std::vector file_schema = { + field_id_col("id", 0, int_type, 0), + field_id_col("score", 1, int_type, 1), + }; + + auto filter_expr = int_gt(table_slot(7, 1, int_type, "score"), 10); + TableFilter filter {.conjunct = VExprContext::create_shared(filter_expr), + .global_indices = {GlobalIndex(1)}, + .can_localize = false}; + + TableColumnMapper mapper({.mode = TableColumnMappingMode::BY_FIELD_ID}); + ASSERT_TRUE(mapper.create_mapping(table_schema, {}, file_schema).ok()); + + FileScanRequest request; + FilterLocalizationResult localization_result; + ASSERT_TRUE(mapper.create_scan_request({filter}, table_schema, &request, nullptr, nullptr, + &localization_result) + .ok()); + + ASSERT_EQ(localization_result.localized_filters, std::vector({false})); + EXPECT_TRUE(request.conjuncts.empty()); + // A TableReader residual still consumes the decoded value, so replacing the hidden payload + // with a predicate-only placeholder would make the fallback expression observe wrong data. + EXPECT_TRUE(request.predicate_only_columns.empty()); +} + // Scenario: a nested predicate targets a table-side renamed struct field; scan projection must // resolve that field to the old physical file child. TEST(ColumnMapperScanRequestTest, NestedPredicateProjectionUsesMappedRenamedChild) { diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 283cee43f5018b..fab2c905dae71c 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -37,7 +37,6 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/field.h" -#include "exec/scan/file_scanner_v2.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "format_v2/column_data.h" @@ -168,42 +167,6 @@ class AppendTrackingTableReader final : public TableReader { size_t owned_conjuncts = 0; }; -class OneRowTableReader final : public TableReader { -public: - Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } - - Status get_block(Block* block, bool* eos) override { - auto column = ColumnInt32::create(); - column->insert_value(1); - block->replace_by_position(0, std::move(column)); - *eos = false; - return Status::OK(); - } -}; - -class StatefulHybridPredicate final : public VExpr { -public: - explicit StatefulHybridPredicate(std::vector* observed_invocations) - : VExpr(std::make_shared(), false), - _observed_invocations(observed_invocations) {} - - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, - ColumnPtr& result_column) const override { - _observed_invocations->push_back(_invocation++); - result_column = ColumnUInt8::create(count, 1); - return Status::OK(); - } - - const std::string& expr_name() const override { return _expr_name; } - bool is_constant() const override { return false; } - bool is_deterministic() const override { return false; } - -private: - std::vector* const _observed_invocations; - mutable int _invocation = 0; - const std::string _expr_name = "StatefulHybridPredicate"; -}; - // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -361,64 +324,11 @@ TEST(HudiHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) { auto literal = VLiteral::create_shared(std::make_shared(), Field::create_field(1)); - ASSERT_TRUE(reader.append_conjuncts_with_ownership( - {VExprContext::create_shared(std::move(literal))}, 0) - .ok()); + ASSERT_TRUE(reader.append_conjuncts({VExprContext::create_shared(std::move(literal))}).ok()); EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); - EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); - EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); -} - -TEST(HudiHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { - RuntimeState state {TQueryOptions(), TQueryGlobals()}; - RuntimeProfile profile("hudi_scanner_stateful_residual"); - TFileScanRangeParams scan_params; - scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); - auto hybrid_reader = std::make_unique(); - auto* hybrid_reader_ptr = hybrid_reader.get(); - hybrid_reader_ptr->TEST_set_child_reader_factories( - [] { return std::make_unique(); }, - [] { return std::make_unique(); }); - - std::vector observed_invocations; - auto conjunct = VExprContext::create_shared( - std::make_shared(&observed_invocations)); - ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); - ASSERT_TRUE(conjunct->open(&state).ok()); - FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); - scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); - - const std::vector projected_columns { - make_table_column(0, "id", std::make_shared()), - }; - ASSERT_TRUE(hybrid_reader_ptr - ->init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = nullptr, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); - - auto run_split = [&](FileFormat format, TFileFormatType::type thrift_format) { - SplitReadOptions split; - split.current_split_format = format; - split.current_range.__set_format_type(thrift_format); - ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); - Block block = build_table_block(projected_columns); - bool eos = false; - ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); - ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); - }; - run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); - run_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); - run_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); - - EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 1); } TEST(HudiHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 41180443419aad..bee6609b5cbac3 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -45,7 +45,6 @@ #include "core/data_type/data_type_string.h" #include "core/field.h" #include "exec/common/endian.h" -#include "exec/scan/file_scanner_v2.h" #include "exprs/vexpr_context.h" #include "exprs/vliteral.h" #include "format/format_common.h" @@ -116,42 +115,6 @@ class AppendTrackingTableReader final : public TableReader { size_t owned_conjuncts = 0; }; -class OneRowTableReader final : public TableReader { -public: - Status prepare_split(const SplitReadOptions&) override { return Status::OK(); } - - Status get_block(Block* block, bool* eos) override { - auto column = ColumnInt32::create(); - column->insert_value(1); - block->replace_by_position(0, std::move(column)); - *eos = false; - return Status::OK(); - } -}; - -class StatefulHybridPredicate final : public VExpr { -public: - explicit StatefulHybridPredicate(std::vector* observed_invocations) - : VExpr(std::make_shared(), false), - _observed_invocations(observed_invocations) {} - - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, - ColumnPtr& result_column) const override { - _observed_invocations->push_back(_invocation++); - result_column = ColumnUInt8::create(count, 1); - return Status::OK(); - } - - const std::string& expr_name() const override { return _expr_name; } - bool is_constant() const override { return false; } - bool is_deterministic() const override { return false; } - -private: - std::vector* const _observed_invocations; - mutable int _invocation = 0; - const std::string _expr_name = "StatefulHybridPredicate"; -}; - DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -846,63 +809,11 @@ TEST(PaimonHybridReaderTest, LateConjunctReachesInitializedNativeAndJniChildren) auto literal = VLiteral::create_shared(std::make_shared(), Field::create_field(1)); - ASSERT_TRUE(reader.append_conjuncts_with_ownership( - {VExprContext::create_shared(std::move(literal))}, 0) - .ok()); + ASSERT_TRUE(reader.append_conjuncts({VExprContext::create_shared(std::move(literal))}).ok()); EXPECT_EQ(native_reader_ptr->appended_conjuncts, 1); EXPECT_EQ(jni_reader_ptr->appended_conjuncts, 1); - EXPECT_EQ(native_reader_ptr->owned_conjuncts, 0); - EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 0); -} - -TEST(PaimonHybridReaderTest, ScannerStatefulResidualSurvivesNativeJniNativeSwitch) { - RuntimeState state {TQueryOptions(), TQueryGlobals()}; - RuntimeProfile profile("paimon_scanner_stateful_residual"); - auto scan_params = make_local_parquet_scan_params(); - auto hybrid_reader = std::make_unique(); - auto* hybrid_reader_ptr = hybrid_reader.get(); - hybrid_reader_ptr->TEST_set_child_reader_factories( - [] { return std::make_unique(); }, - [] { return std::make_unique(); }); - - std::vector observed_invocations; - auto conjunct = VExprContext::create_shared( - std::make_shared(&observed_invocations)); - ASSERT_TRUE(conjunct->prepare(&state, RowDescriptor {}).ok()); - ASSERT_TRUE(conjunct->open(&state).ok()); - FileScannerV2 scanner(&state, &profile, std::move(hybrid_reader)); - scanner.TEST_set_scanner_conjuncts({std::move(conjunct)}); - - const std::vector projected_columns { - make_table_column(0, "id", std::make_shared()), - }; - ASSERT_TRUE(hybrid_reader_ptr - ->init({ - .projected_columns = projected_columns, - .conjuncts = {}, - .format = FileFormat::PARQUET, - .scan_params = &scan_params, - .io_ctx = nullptr, - .runtime_state = &state, - .scanner_profile = &profile, - }) - .ok()); - - auto run_split = [&](FileFormat format, TFileRangeDesc range) { - SplitReadOptions split; - split.current_split_format = format; - split.current_range = std::move(range); - ASSERT_TRUE(hybrid_reader_ptr->prepare_split(split).ok()); - Block block = build_table_block(projected_columns); - bool eos = false; - ASSERT_TRUE(hybrid_reader_ptr->get_block(&block, &eos).ok()); - ASSERT_TRUE(scanner.TEST_filter_output_block(&block).ok()); - }; - run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); - run_split(FileFormat::JNI, make_paimon_jni_range()); - run_split(FileFormat::PARQUET, make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); - - EXPECT_EQ(observed_invocations, std::vector({0, 1, 2})); + EXPECT_EQ(native_reader_ptr->owned_conjuncts, 1); + EXPECT_EQ(jni_reader_ptr->owned_conjuncts, 1); } TEST(PaimonHybridReaderTest, AggregatesConditionCacheHitsFromBothChildren) { diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 4d57ea0b7ccdb1..43f1ccedbc7f06 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1457,7 +1457,7 @@ TEST(TableReaderTest, PrepareSplitPrunesFileBackedIdentityPartitionRuntimeFilter EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 1); } -TEST(TableReaderTest, PrepareSplitDoesNotEvaluateNonDeterministicPartitionPredicate) { +TEST(TableReaderTest, PrepareSplitSkipsNonDeterministicAndChecksLaterSafePredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1491,12 +1491,12 @@ TEST(TableReaderTest, PrepareSplitDoesNotEvaluateNonDeterministicPartitionPredic ASSERT_TRUE(reader.prepare_split(split).ok()); EXPECT_FALSE(predicate_executed); - EXPECT_FALSE(reader.current_split_pruned()); + EXPECT_TRUE(reader.current_split_pruned()); ASSERT_NE(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum"), nullptr); - EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 0); + EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 1); } -TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { +TEST(TableReaderTest, ConstantPruningSkipsUnsafeAndChecksLaterSafePredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1535,17 +1535,15 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_TRUE(predicate_executed); - EXPECT_FALSE(eos); - // The file was still opened, proving constant pruning did not jump over the unsafe predicate; - // the predicate is evaluated only after the resulting table row is materialized. - EXPECT_EQ(fake_state->open_count, 1); - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(predicate_executed); EXPECT_TRUE(eos); + // An unsafe expression is not evaluated speculatively, but it is no longer an ordering + // barrier for an independent safe predicate that can prove the split is empty. + EXPECT_EQ(fake_state->open_count, 0); ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, UnsafePredicateRunsAfterTableMaterialization) { +TEST(TableReaderTest, MixedPredicatesKeepOnlyUnsafePredicateAtTableLayer) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); std::vector projected_columns; @@ -1561,7 +1559,13 @@ TEST(TableReaderTest, UnsafePredicateRunsAfterTableMaterialization) { FakeTableReader reader(file_schema, fake_state); ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, - .conjuncts = {prepared_conjunct(&state, unsafe_predicate)}, + .conjuncts = + { + prepared_conjunct(&state, unsafe_predicate), + prepared_conjunct( + &state, + table_int32_greater_than_expr(0, 0, 0)), + }, .format = FileFormat::PARQUET, .scan_params = nullptr, .io_ctx = nullptr, @@ -1577,8 +1581,10 @@ TEST(TableReaderTest, UnsafePredicateRunsAfterTableMaterialization) { bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_NE(fake_state->last_request, nullptr); - EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_EQ(fake_state->last_request->conjuncts.size(), 1); + EXPECT_TRUE(fake_state->last_request->conjuncts.front()->root()->is_deterministic()); EXPECT_TRUE(predicate_executed); + EXPECT_EQ(block.rows(), 0); ASSERT_TRUE(reader.close().ok()); } @@ -1839,7 +1845,7 @@ TEST(TableReaderTest, ResidualFilteringHasDedicatedProfileTimer) { ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { +TEST(TableReaderTest, ConstantPruningSkipsUnsafeSlotlessPredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1869,28 +1875,22 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { .scanner_profile = nullptr, }) .ok()); + // Opening a slotless VExpr caches its constant result. Reset the observation so the assertions + // below distinguish split pruning from normal expression-context initialization. + predicate_executed = false; SplitReadOptions split; split.current_range.__set_path("fake-table-reader-input"); split.partition_values.emplace("part", Field::create_field(7)); ASSERT_TRUE(reader.prepare_split(split).ok()); + EXPECT_FALSE(predicate_executed); Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_TRUE(predicate_executed); - EXPECT_FALSE(eos); - // The later partition predicate is false for part=7. Opening the file proves constant pruning - // stopped at the earlier unsafe expression even though that expression had no slot and thus no - // entry in `_table_filters`. - EXPECT_EQ(fake_state->open_count, 1); - ASSERT_NE(fake_state->last_request, nullptr); - // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. - // The later predicate must stay on the post-materialization path instead of running inside the - // file reader before the unsafe conjunct. - EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(predicate_executed); EXPECT_TRUE(eos); + EXPECT_EQ(fake_state->open_count, 0); ASSERT_TRUE(reader.close().ok()); } From f24f78aa8d4a2453bf0830f55c9d381685ca0dbc Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 30 Jul 2026 23:03:34 +0800 Subject: [PATCH 08/10] [improvement](scan) Avoid materializing post-filter predicate columns Pass post-filter slot liveness from FE while keeping exact predicate localization a split-local BE decision. Preserve residual and unsafe predicate ordering so payloads are discarded only when all referencing predicates execute in the file reader. --- be/src/exec/scan/file_scanner_v2.cpp | 5 + be/src/format_v2/column_data.h | 8 +- be/src/format_v2/column_mapper.cpp | 41 +++---- be/src/format_v2/column_mapper.h | 1 + be/src/format_v2/table_reader.cpp | 22 ++-- be/src/format_v2/table_reader.h | 21 +++- be/test/format_v2/table_reader_test.cpp | 104 ++++++++++++++---- .../doris/datasource/FileQueryScanNode.java | 14 +++ .../translator/PhysicalPlanTranslator.java | 8 ++ .../datasource/FileQueryScanNodeTest.java | 28 +++++ gensrc/thrift/PlanNodes.thrift | 4 + 11 files changed, 201 insertions(+), 55 deletions(-) diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index e2443afa991631..fcee52fd95315a 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -789,6 +789,11 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_ slot_info.slot_id); } auto column = _build_table_column(it->second); + // An old FE does not send post-filter liveness, so preserve the historical behavior and + // materialize the value. A new FE may mark it dead, but ColumnMapper still requires exact + // localization for the current split before making it predicate-only. + column.value_required_after_filter = !slot_info.__isset.value_required_after_filter || + slot_info.value_required_after_filter; build_context.slot_desc = it->second; if (column.name.starts_with(BeConsts::GLOBAL_ROWID_COL)) { _need_global_rowid_column = true; diff --git a/be/src/format_v2/column_data.h b/be/src/format_v2/column_data.h index d504305fad4014..4476315ca754eb 100644 --- a/be/src/format_v2/column_data.h +++ b/be/src/format_v2/column_data.h @@ -223,9 +223,9 @@ struct GlobalRowIdContext { }; // Column schema definition shared by table/global projection and file-local schema matching. -// -// ColumnDefinition intentionally carries schema identity only. FE column unique ids are translated -// to GlobalIndex at the FileScannerV2 boundary and must not appear in table/file reader APIs. +// It also carries scan-projection metadata whose meaning is independent of FE descriptor ids. +// FE column unique ids are translated to GlobalIndex at the FileScannerV2 boundary and must not +// appear in table/file reader APIs. struct ColumnDefinition { // Typed identifier value used to match a column against another schema. // @@ -273,6 +273,8 @@ struct ColumnDefinition { // Partition columns are constants from split metadata and should not be matched against file // schema unless table-format logic explicitly asks for it. bool is_partition_key = false; + // True when the table-level value is consumed after scan predicates have been evaluated. + bool value_required_after_filter = true; // File-local column kind. For table/global columns this remains DATA_COLUMN. ColumnType column_type = ColumnType::DATA_COLUMN; diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index aabe0b8ea91591..804e7c7ed154f1 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -439,7 +439,8 @@ std::string ColumnDefinition::debug_string() const { << join_debug_strings(identity_children, [](const ColumnDefinition& child) { return child.debug_string(); }) << ", has_default_expr=" << (default_expr != nullptr) - << ", is_partition_key=" << is_partition_key << "}"; + << ", is_partition_key=" << is_partition_key + << ", value_required_after_filter=" << value_required_after_filter << "}"; return out.str(); } @@ -480,6 +481,7 @@ std::string ColumnMapping::debug_string() const { [](const ColumnMapping& child) { return child.debug_string(); }) << ", is_trivial=" << is_trivial << ", is_constant=" << constant_index.has_value() << ", filter_conversion=" << filter_conversion_type_to_string(filter_conversion) + << ", value_required_after_filter=" << value_required_after_filter << ", virtual_column_type=" << virtual_column_type_to_string(virtual_column_type) << ", has_default_expr=" << (default_expr != nullptr) << "}"; return out.str(); @@ -2031,6 +2033,7 @@ Status TableColumnMapper::_create_mapping_for_column(const ColumnDefinition& tab mapping->global_index = global_index; mapping->table_column_name = table_column.name; mapping->table_type = table_column.type; + mapping->value_required_after_filter = table_column.value_required_after_filter; // Row-lineage names are Iceberg metadata contracts, not reserved names in generic Hive, // Hudi, or Paimon schemas. Only the Iceberg reader may opt into virtual synthesis. const auto row_lineage_type = @@ -2237,39 +2240,39 @@ Status TableColumnMapper::create_scan_request( localization_result == nullptr ? &local_localization_result : localization_result; RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state, exact_localization_result)); - for (const auto& mapping : _hidden_mappings) { - if (!mapping.file_local_id.has_value()) { - continue; - } - const auto local_id = LocalColumnId(*mapping.file_local_id); - const bool is_visible_output = + for (const auto& predicate_column : file_request->predicate_columns) { + const auto local_id = predicate_column.column_id(); + const bool value_required_after_filter = std::ranges::any_of(_mappings, [local_id](const ColumnMapping& visible_mapping) { return visible_mapping.file_local_id.has_value() && - LocalColumnId(*visible_mapping.file_local_id) == local_id; + LocalColumnId(*visible_mapping.file_local_id) == local_id && + visible_mapping.value_required_after_filter; }); - if (is_visible_output) { + if (value_required_after_filter) { continue; } bool referenced_by_filter = false; bool referenced_only_by_localized_filters = true; for (size_t filter_index = 0; filter_index < table_filters.size(); ++filter_index) { - if (std::ranges::find(table_filters[filter_index].global_indices, - mapping.global_index) == - table_filters[filter_index].global_indices.end()) { + const bool references_local_column = std::ranges::any_of( + table_filters[filter_index].global_indices, [&](GlobalIndex global_index) { + const auto* mapping = _find_filter_mapping(global_index); + return mapping != nullptr && mapping->file_local_id.has_value() && + LocalColumnId(*mapping->file_local_id) == local_id; + }); + if (!references_local_column) { continue; } referenced_by_filter = true; referenced_only_by_localized_filters &= exact_localization_result->localized_filters[filter_index]; } - // A localized predicate is enforced exactly before TableReader materializes output. Only - // hidden columns used exclusively by exact FileReader predicates may discard their payload. - // A TableReader residual still needs its value after table-schema materialization. + // A scan tuple can contain a filter slot that its upstream projection does not consume. + // For example, `SELECT SUM(measure) FROM t WHERE filter_key BETWEEN 1 AND 20` needs + // `filter_key` while evaluating the predicate, but not afterwards. Discard its payload + // only when every predicate referencing the physical column was localized exactly for this + // split; schema evolution or a TableReader residual must retain the value. if (referenced_by_filter && referenced_only_by_localized_filters && - std::ranges::any_of(file_request->predicate_columns, - [local_id](const LocalColumnIndex& projection) { - return projection.column_id() == local_id; - }) && !file_request->is_predicate_only(local_id)) { file_request->predicate_only_columns.push_back(local_id); } diff --git a/be/src/format_v2/column_mapper.h b/be/src/format_v2/column_mapper.h index a6aaa342ad8cb9..244c9893fd765e 100644 --- a/be/src/format_v2/column_mapper.h +++ b/be/src/format_v2/column_mapper.h @@ -155,6 +155,7 @@ struct ColumnMapping { // How filters referencing this table/global column can be converted below table-reader // finalize. This is metadata for localize_filters() and future constant-filter evaluation. FilterConversionType filter_conversion = FilterConversionType::FINALIZE_ONLY; + bool value_required_after_filter = true; TableVirtualColumnType virtual_column_type = TableVirtualColumnType::INVALID; VExprContextSPtr default_expr; // One-row constant owns variable-width payloads; Field is only a borrowed diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 3262bd407f36ed..a2c232177295eb 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -822,21 +822,27 @@ Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { Status TableReader::_build_table_filters_from_conjuncts() { _table_filters.clear(); + _constant_pruning_safe_filter_count = 0; + bool in_safe_prefix = true; for (size_t conjunct_index = 0; conjunct_index < _conjuncts.size(); ++conjunct_index) { const auto& conjunct = _conjuncts[conjunct_index]; DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); - const bool can_localize = is_safe_to_pre_execute(conjunct); + // Do not move a later predicate ahead of an unsafe, stateful, or error-preserving + // conjunct. Even a slotless conjunct remains an execution-order barrier. + if (in_safe_prefix && !is_safe_to_pre_execute(conjunct)) { + in_safe_prefix = false; + } const size_t filters_before = _table_filters.size(); RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); for (size_t filter_index = filters_before; filter_index < _table_filters.size(); ++filter_index) { _table_filters[filter_index].source_conjunct_index = conjunct_index; - // Each predicate owns its execution layer independently. An unsafe predicate remains - // at TableReader, but it must not prevent a later safe predicate from becoming an - // exact FileReader predicate. - _table_filters[filter_index].can_localize = can_localize; + _table_filters[filter_index].can_localize = in_safe_prefix; + } + if (in_safe_prefix) { + _constant_pruning_safe_filter_count = _table_filters.size(); } } return Status::OK(); @@ -1324,10 +1330,10 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& for (const auto& conjunct : conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); - // Unsafe or non-deterministic predicates remain at TableReader. Safe predicates are - // independent pruning candidates even when an earlier predicate cannot be pre-executed. + // Keep the safe prefix of the original order. Pruning with a later predicate must not skip + // evaluation of an earlier stateful, non-deterministic, or error-preserving conjunct. if (!is_safe_to_pre_execute(conjunct)) { - continue; + break; } std::set global_indices; collect_global_indices(conjunct->root(), &global_indices); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index 6db5a5de84a172..cbf2d3d3cba096 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -83,8 +83,8 @@ struct TableFilter { VExprContextSPtr conjunct; std::vector global_indices; size_t source_conjunct_index = 0; - // Only safe and deterministic predicates may move below table-schema materialization. - // Unsafe, non-deterministic, or unlocalizable predicates remain exact TableReader residuals. + // False after the first unsafe source conjunct so file-local execution cannot reorder a later + // predicate ahead of stateful or error-preserving table semantics. bool can_localize = true; }; @@ -262,8 +262,8 @@ class TableReader { Status append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, size_t table_reader_owned_conjunct_count); - // Shared safety classification for deciding whether one predicate may execute before - // table-schema materialization without changing stateful or error-preserving semantics. + // Shared safety classification for deciding which ordered conjunct prefix may execute below + // Scanner without changing stateful or error-preserving semantics. static bool is_safe_to_pre_execute(const VExprContextSPtr& conjunct); virtual const MaterializedBlockStats& last_materialized_block_stats() const { @@ -596,9 +596,14 @@ class TableReader { Status _evaluate_constant_filters(bool* can_filter_all) { DORIS_CHECK(can_filter_all != nullptr); + DORIS_CHECK_LE(_constant_pruning_safe_filter_count, _table_filters.size()); *can_filter_all = false; - for (const auto& table_filter : _table_filters) { - if (table_filter.conjunct == nullptr || !table_filter.can_localize) { + // The bound comes from the original conjunct order, including slotless expressions that + // are absent from _table_filters but still form execution-order barriers. + for (size_t filter_index = 0; filter_index < _constant_pruning_safe_filter_count; + ++filter_index) { + const auto& table_filter = _table_filters[filter_index]; + if (table_filter.conjunct == nullptr) { continue; } DORIS_CHECK(is_safe_to_pre_execute(table_filter.conjunct)); @@ -817,6 +822,7 @@ class TableReader { _data_reader.column_mapper.reset(); } _table_filters.clear(); + _constant_pruning_safe_filter_count = 0; _remaining_conjuncts.clear(); _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); @@ -1946,6 +1952,9 @@ class TableReader { std::map _partition_values; // Predicates built from scan conjuncts before file-level localization. std::vector _table_filters; + // Number of filters before the first unsafe original conjunct. Slotless conjuncts are omitted + // from _table_filters but still stop this safe prefix. + size_t _constant_pruning_safe_filter_count = 0; VExprContextSPtrs _conjuncts; size_t _table_reader_owned_conjunct_count = 0; std::optional _appended_table_reader_owned_conjunct_count; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 43f1ccedbc7f06..0c0109951ab283 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1457,7 +1457,7 @@ TEST(TableReaderTest, PrepareSplitPrunesFileBackedIdentityPartitionRuntimeFilter EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 1); } -TEST(TableReaderTest, PrepareSplitSkipsNonDeterministicAndChecksLaterSafePredicate) { +TEST(TableReaderTest, PrepareSplitStopsPruningAtNonDeterministicPredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1491,12 +1491,12 @@ TEST(TableReaderTest, PrepareSplitSkipsNonDeterministicAndChecksLaterSafePredica ASSERT_TRUE(reader.prepare_split(split).ok()); EXPECT_FALSE(predicate_executed); - EXPECT_TRUE(reader.current_split_pruned()); + EXPECT_FALSE(reader.current_split_pruned()); ASSERT_NE(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum"), nullptr); - EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 1); + EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 0); } -TEST(TableReaderTest, ConstantPruningSkipsUnsafeAndChecksLaterSafePredicate) { +TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1535,15 +1535,17 @@ TEST(TableReaderTest, ConstantPruningSkipsUnsafeAndChecksLaterSafePredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_FALSE(predicate_executed); + EXPECT_TRUE(predicate_executed); + EXPECT_FALSE(eos); + // The file was still opened, proving constant pruning did not jump over the unsafe predicate; + // the predicate is evaluated only after the resulting table row is materialized. + EXPECT_EQ(fake_state->open_count, 1); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_TRUE(eos); - // An unsafe expression is not evaluated speculatively, but it is no longer an ordering - // barrier for an independent safe predicate that can prove the split is empty. - EXPECT_EQ(fake_state->open_count, 0); ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, MixedPredicatesKeepOnlyUnsafePredicateAtTableLayer) { +TEST(TableReaderTest, MixedPredicatesStayAtTableLayerAfterUnsafePredicate) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); std::vector projected_columns; @@ -1581,8 +1583,7 @@ TEST(TableReaderTest, MixedPredicatesKeepOnlyUnsafePredicateAtTableLayer) { bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_NE(fake_state->last_request, nullptr); - ASSERT_EQ(fake_state->last_request->conjuncts.size(), 1); - EXPECT_TRUE(fake_state->last_request->conjuncts.front()->root()->is_deterministic()); + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); EXPECT_TRUE(predicate_executed); EXPECT_EQ(block.rows(), 0); ASSERT_TRUE(reader.close().ok()); @@ -1845,7 +1846,7 @@ TEST(TableReaderTest, ResidualFilteringHasDedicatedProfileTimer) { ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, ConstantPruningSkipsUnsafeSlotlessPredicate) { +TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1875,22 +1876,27 @@ TEST(TableReaderTest, ConstantPruningSkipsUnsafeSlotlessPredicate) { .scanner_profile = nullptr, }) .ok()); - // Opening a slotless VExpr caches its constant result. Reset the observation so the assertions - // below distinguish split pruning from normal expression-context initialization. - predicate_executed = false; - SplitReadOptions split; split.current_range.__set_path("fake-table-reader-input"); split.partition_values.emplace("part", Field::create_field(7)); ASSERT_TRUE(reader.prepare_split(split).ok()); - EXPECT_FALSE(predicate_executed); Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_FALSE(predicate_executed); + EXPECT_TRUE(predicate_executed); + EXPECT_FALSE(eos); + // The later partition predicate is false for part=7. Opening the file proves constant pruning + // stopped at the earlier unsafe expression even though that expression had no slot and thus no + // entry in `_table_filters`. + EXPECT_EQ(fake_state->open_count, 1); + ASSERT_NE(fake_state->last_request, nullptr); + // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. + // The later predicate must stay on the post-materialization path instead of running inside the + // file reader before the unsafe conjunct. + EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_TRUE(eos); - EXPECT_EQ(fake_state->open_count, 0); ASSERT_TRUE(reader.close().ok()); } @@ -5541,6 +5547,66 @@ TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 0); } +TEST(TableReaderTest, CreateScanRequestDiscardsVisiblePredicatePayloadNotNeededAfterFilter) { + const auto int_type = std::make_shared(); + std::vector projected_columns = { + make_table_column(0, "filter_key", int_type), + make_table_column(1, "measure", int_type), + }; + projected_columns[0].value_required_after_filter = false; + const std::vector file_schema = { + make_file_column(0, "filter_key", int_type), + make_file_column(1, "measure", int_type), + }; + + TableColumnMapper mapper; + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, file_schema).ok()); + + TableFilter table_filter { + .conjunct = VExprContext::create_shared(table_int32_greater_than_expr(0, 0, 1)), + .global_indices = {GlobalIndex(0)}, + }; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({table_filter}, projected_columns, &file_request).ok()); + + EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); + EXPECT_EQ(file_request.predicate_only_columns, std::vector({LocalColumnId(0)})); +} + +TEST(TableReaderTest, CreateScanRequestKeepsPayloadWhenAnyReferencingFilterIsResidual) { + const auto int_type = std::make_shared(); + std::vector projected_columns = { + make_table_column(0, "filter_key", int_type), + make_table_column(1, "measure", int_type), + }; + projected_columns[0].value_required_after_filter = false; + const std::vector file_schema = { + make_file_column(0, "filter_key", int_type), + make_file_column(1, "measure", int_type), + }; + + TableColumnMapper mapper; + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, file_schema).ok()); + + TableFilter localized_filter { + .conjunct = VExprContext::create_shared(table_int32_greater_than_expr(0, 0, 1)), + .global_indices = {GlobalIndex(0)}, + }; + TableFilter residual_filter { + .conjunct = VExprContext::create_shared(table_int32_greater_than_expr(0, 0, 2)), + .global_indices = {GlobalIndex(0)}, + .can_localize = false, + }; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({localized_filter, residual_filter}, projected_columns, + &file_request) + .ok()); + + EXPECT_TRUE(file_request.predicate_only_columns.empty()); +} + TEST(TableReaderTest, CreateScanRequestUsesColumnNameForByNamePredicateMapping) { const auto int_type = std::make_shared(); std::vector projected_columns = { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 873ab200350297..ea9007c2ff4c4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource; import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TableSample; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; @@ -115,6 +116,9 @@ public abstract class FileQueryScanNode extends FileScanNode { protected FileSplitter fileSplitter; protected SummaryProfile summaryProfile; + // Null means the planner did not provide post-filter liveness, so all slots remain required. + private Set slotsRequiredAfterFilter; + // The data cache function only works for queries on Hive, Iceberg, Hudi(via HMS), and Paimon tables. // See: https://doris.incubator.apache.org/docs/dev/lakehouse/data-cache private static final Set CACHEABLE_CATALOGS = new HashSet<>( @@ -197,6 +201,7 @@ protected void initSchemaParams() throws UserException { TColumnCategory category = classifyColumn(slot, partitionKeys); slotInfo.setCategory(category); slotInfo.setIsFileSlot(isFileSlot(category)); + slotInfo.setValueRequiredAfterFilter(isValueRequiredAfterFilter(slot)); params.addToRequiredSlots(slotInfo); } // Defaults are field semantics, so resolve them from the same relation schema as the @@ -231,12 +236,21 @@ private void updateRequiredSlots() throws UserException { TColumnCategory category = classifyColumn(slot, partitionKeys); slotInfo.setCategory(category); slotInfo.setIsFileSlot(isFileSlot(category)); + slotInfo.setValueRequiredAfterFilter(isValueRequiredAfterFilter(slot)); params.addToRequiredSlots(slotInfo); } // Update required slots and column_idxs in scanRangeLocations. setColumnPositionMapping(); } + public void setSlotsRequiredAfterFilter(Set slotIds) { + slotsRequiredAfterFilter = new HashSet<>(slotIds); + } + + private boolean isValueRequiredAfterFilter(SlotDescriptor slot) { + return slotsRequiredAfterFilter == null || slotsRequiredAfterFilter.contains(slot.getId()); + } + /** * Classify a column's category for the BE reader. * Subclasses override this for format-specific classification. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index a704909a9f35dd..e1cfd82f8a2b09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -3082,6 +3082,7 @@ private void updateScanSlotsMaterialization(ScanNode scanNode, Set requiredSlotIdSet, Set requiredByProjectSlotIdSet, PlanTranslatorContext context) { Set requiredWithVirtualColumns = Sets.newHashSet(requiredSlotIdSet); + Set requiredAfterFilterWithVirtualColumns = Sets.newHashSet(requiredByProjectSlotIdSet); for (SlotDescriptor virtualSlot : scanNode.getTupleDesc().getSlots()) { Expr virtualColumn = virtualSlot.getVirtualColumn(); if (virtualColumn == null) { @@ -3095,6 +3096,13 @@ private void updateScanSlotsMaterialization(ScanNode scanNode, .map(SlotRef::getSlotId) .collect(Collectors.toSet()); requiredWithVirtualColumns.addAll(virtualColumnInputSlotIds); + requiredAfterFilterWithVirtualColumns.addAll(virtualColumnInputSlotIds); + } + if (scanNode instanceof FileQueryScanNode) { + // FE reports only projection liveness. Schema evolution and exact predicate + // localization remain split-local decisions made by BE's column mapper. + ((FileQueryScanNode) scanNode).setSlotsRequiredAfterFilter( + requiredAfterFilterWithVirtualColumns); } // Find the smallest column, for count(*) or other situation that slot is empty after prune SlotDescriptor smallest = getSmallestSlot(scanNode.getTupleDesc().getSlots()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index f7a55ae2f6c3c4..a59a97a6950a3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; public class FileQueryScanNodeTest { private static final long MB = 1024L * 1024L; @@ -168,6 +169,33 @@ public void testUpdateRequiredSlotsPreservesInlineDefaultValueExpr() throws Exce Assert.assertSame(slotInfo, updatedSlotInfo); Assert.assertTrue(updatedSlotInfo.isSetDefaultValueExpr()); Assert.assertSame(defaultExpr, updatedSlotInfo.getDefaultValueExpr()); + Assert.assertTrue(updatedSlotInfo.isValueRequiredAfterFilter()); + } + + @Test + public void testUpdateRequiredSlotsMarksPostFilterValueLiveness() throws Exception { + SessionVariable sv = new SessionVariable(); + TestFileQueryScanNode node = new TestFileQueryScanNode(sv); + node.setTargetTable(table); + + TupleDescriptor desc = node.getTupleDescriptor(); + desc.setTable(table); + SlotDescriptor filterSlot = new SlotDescriptor(new SlotId(1), desc); + filterSlot.setColumn(new Column("filter_key", Type.INT)); + desc.addSlot(filterSlot); + SlotDescriptor outputSlot = new SlotDescriptor(new SlotId(2), desc); + outputSlot.setColumn(new Column("measure", Type.INT)); + desc.addSlot(outputSlot); + Mockito.when(table.getFullSchema()).thenReturn(Arrays.asList( + filterSlot.getColumn(), outputSlot.getColumn())); + + node.setSlotsRequiredAfterFilter(Collections.singleton(outputSlot.getId())); + node.params = new TFileScanRangeParams(); + + UPDATE_REQUIRED_SLOTS_METHOD.invoke(node); + + Assert.assertFalse(node.params.getRequiredSlots().get(0).isValueRequiredAfterFilter()); + Assert.assertTrue(node.params.getRequiredSlots().get(1).isValueRequiredAfterFilter()); } @Test diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index a76e4543b38f30..1849ff1dbe9c82 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -272,6 +272,10 @@ struct TFileScanSlotInfo { // Populated by FE from Column.getDefaultValue() or NULL literal. // This replaces the separate default_value_of_src_slot map in TFileScanRangeParams. 4: optional Exprs.TExpr default_value_expr; + // Whether an operator above the scan consumes this slot after scan predicates are evaluated. + // This is only liveness metadata; BE still decides per split whether every referencing + // predicate was localized exactly before it can discard the column payload. + 5: optional bool value_required_after_filter; } // descirbe how to read file From 1e6d0f3b84cc9be28c409824ff898739998d7682 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 2 Aug 2026 12:13:58 +0800 Subject: [PATCH 09/10] [fix](build) Remove unused FileQueryScanNodeTest import --- .../java/org/apache/doris/datasource/FileQueryScanNodeTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java index a59a97a6950a3b..27996b6db48b9e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FileQueryScanNodeTest.java @@ -49,7 +49,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; public class FileQueryScanNodeTest { private static final long MB = 1024L * 1024L; From e3d2b37edac5f3ee0717457a9a904d1b1ec8d647 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 31 Jul 2026 08:26:50 +0800 Subject: [PATCH 10/10] [fix](scan) Keep predicate localization split-local Restore independent predicate localization and add coverage showing that a predicate localized for one split returns to TableReader when the next split cannot localize it. Residual predicates on unrelated physical columns do not prevent predicate-only payload discard. --- be/src/format_v2/table_reader.cpp | 22 ++--- be/src/format_v2/table_reader.h | 21 ++--- be/test/format_v2/table_reader_test.cpp | 118 +++++++++++++++++------- 3 files changed, 101 insertions(+), 60 deletions(-) diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index a2c232177295eb..3262bd407f36ed 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -822,27 +822,21 @@ Status TableReader::append_conjuncts(const VExprContextSPtrs& conjuncts) { Status TableReader::_build_table_filters_from_conjuncts() { _table_filters.clear(); - _constant_pruning_safe_filter_count = 0; - bool in_safe_prefix = true; for (size_t conjunct_index = 0; conjunct_index < _conjuncts.size(); ++conjunct_index) { const auto& conjunct = _conjuncts[conjunct_index]; DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); - // Do not move a later predicate ahead of an unsafe, stateful, or error-preserving - // conjunct. Even a slotless conjunct remains an execution-order barrier. - if (in_safe_prefix && !is_safe_to_pre_execute(conjunct)) { - in_safe_prefix = false; - } + const bool can_localize = is_safe_to_pre_execute(conjunct); const size_t filters_before = _table_filters.size(); RETURN_IF_ERROR( build_table_filters_from_conjunct(conjunct, _runtime_state, &_table_filters)); for (size_t filter_index = filters_before; filter_index < _table_filters.size(); ++filter_index) { _table_filters[filter_index].source_conjunct_index = conjunct_index; - _table_filters[filter_index].can_localize = in_safe_prefix; - } - if (in_safe_prefix) { - _constant_pruning_safe_filter_count = _table_filters.size(); + // Each predicate owns its execution layer independently. An unsafe predicate remains + // at TableReader, but it must not prevent a later safe predicate from becoming an + // exact FileReader predicate. + _table_filters[filter_index].can_localize = can_localize; } } return Status::OK(); @@ -1330,10 +1324,10 @@ Status TableReader::_evaluate_partition_prune_conjuncts(const VExprContextSPtrs& for (const auto& conjunct : conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); - // Keep the safe prefix of the original order. Pruning with a later predicate must not skip - // evaluation of an earlier stateful, non-deterministic, or error-preserving conjunct. + // Unsafe or non-deterministic predicates remain at TableReader. Safe predicates are + // independent pruning candidates even when an earlier predicate cannot be pre-executed. if (!is_safe_to_pre_execute(conjunct)) { - break; + continue; } std::set global_indices; collect_global_indices(conjunct->root(), &global_indices); diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index cbf2d3d3cba096..6db5a5de84a172 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -83,8 +83,8 @@ struct TableFilter { VExprContextSPtr conjunct; std::vector global_indices; size_t source_conjunct_index = 0; - // False after the first unsafe source conjunct so file-local execution cannot reorder a later - // predicate ahead of stateful or error-preserving table semantics. + // Only safe and deterministic predicates may move below table-schema materialization. + // Unsafe, non-deterministic, or unlocalizable predicates remain exact TableReader residuals. bool can_localize = true; }; @@ -262,8 +262,8 @@ class TableReader { Status append_conjuncts_with_ownership(const VExprContextSPtrs& conjuncts, size_t table_reader_owned_conjunct_count); - // Shared safety classification for deciding which ordered conjunct prefix may execute below - // Scanner without changing stateful or error-preserving semantics. + // Shared safety classification for deciding whether one predicate may execute before + // table-schema materialization without changing stateful or error-preserving semantics. static bool is_safe_to_pre_execute(const VExprContextSPtr& conjunct); virtual const MaterializedBlockStats& last_materialized_block_stats() const { @@ -596,14 +596,9 @@ class TableReader { Status _evaluate_constant_filters(bool* can_filter_all) { DORIS_CHECK(can_filter_all != nullptr); - DORIS_CHECK_LE(_constant_pruning_safe_filter_count, _table_filters.size()); *can_filter_all = false; - // The bound comes from the original conjunct order, including slotless expressions that - // are absent from _table_filters but still form execution-order barriers. - for (size_t filter_index = 0; filter_index < _constant_pruning_safe_filter_count; - ++filter_index) { - const auto& table_filter = _table_filters[filter_index]; - if (table_filter.conjunct == nullptr) { + for (const auto& table_filter : _table_filters) { + if (table_filter.conjunct == nullptr || !table_filter.can_localize) { continue; } DORIS_CHECK(is_safe_to_pre_execute(table_filter.conjunct)); @@ -822,7 +817,6 @@ class TableReader { _data_reader.column_mapper.reset(); } _table_filters.clear(); - _constant_pruning_safe_filter_count = 0; _remaining_conjuncts.clear(); _data_reader.file_schema.clear(); _data_reader.file_block_layout.clear(); @@ -1952,9 +1946,6 @@ class TableReader { std::map _partition_values; // Predicates built from scan conjuncts before file-level localization. std::vector _table_filters; - // Number of filters before the first unsafe original conjunct. Slotless conjuncts are omitted - // from _table_filters but still stop this safe prefix. - size_t _constant_pruning_safe_filter_count = 0; VExprContextSPtrs _conjuncts; size_t _table_reader_owned_conjunct_count = 0; std::optional _appended_table_reader_owned_conjunct_count; diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 0c0109951ab283..922eaa89094828 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -1367,6 +1367,24 @@ class FakeTableReader final : public TableReader { std::shared_ptr _state; }; +class SplitLocalPredicateTableReader final : public TableReader { +public: + const std::vector>& predicate_layer_counts() const { + return _predicate_layer_counts; + } + +protected: + Status customize_file_scan_request(FileScanRequest* file_request) override { + DORIS_CHECK(file_request != nullptr); + _predicate_layer_counts.emplace_back(file_request->conjuncts.size(), + _remaining_conjuncts.size()); + return TableReader::customize_file_scan_request(file_request); + } + +private: + std::vector> _predicate_layer_counts; +}; + class ScopedConditionCacheForTest { public: ScopedConditionCacheForTest() @@ -1457,7 +1475,7 @@ TEST(TableReaderTest, PrepareSplitPrunesFileBackedIdentityPartitionRuntimeFilter EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 1); } -TEST(TableReaderTest, PrepareSplitStopsPruningAtNonDeterministicPredicate) { +TEST(TableReaderTest, PrepareSplitSkipsNonDeterministicAndChecksLaterSafePredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1491,12 +1509,12 @@ TEST(TableReaderTest, PrepareSplitStopsPruningAtNonDeterministicPredicate) { ASSERT_TRUE(reader.prepare_split(split).ok()); EXPECT_FALSE(predicate_executed); - EXPECT_FALSE(reader.current_split_pruned()); + EXPECT_TRUE(reader.current_split_pruned()); ASSERT_NE(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum"), nullptr); - EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 0); + EXPECT_EQ(profile.get_counter("RuntimeFilterPartitionPrunedRangeNum")->value(), 1); } -TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { +TEST(TableReaderTest, ConstantPruningSkipsUnsafeAndChecksLaterSafePredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1535,17 +1553,15 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafePredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_TRUE(predicate_executed); - EXPECT_FALSE(eos); - // The file was still opened, proving constant pruning did not jump over the unsafe predicate; - // the predicate is evaluated only after the resulting table row is materialized. - EXPECT_EQ(fake_state->open_count, 1); - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(predicate_executed); EXPECT_TRUE(eos); + // An unsafe expression is not evaluated speculatively, but it is no longer an ordering + // barrier for an independent safe predicate that can prove the split is empty. + EXPECT_EQ(fake_state->open_count, 0); ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, MixedPredicatesStayAtTableLayerAfterUnsafePredicate) { +TEST(TableReaderTest, MixedPredicatesKeepOnlyUnsafePredicateAtTableLayer) { std::vector file_schema; file_schema.push_back(make_file_column(0, "id", std::make_shared())); std::vector projected_columns; @@ -1583,7 +1599,8 @@ TEST(TableReaderTest, MixedPredicatesStayAtTableLayerAfterUnsafePredicate) { bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_NE(fake_state->last_request, nullptr); - EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); + ASSERT_EQ(fake_state->last_request->conjuncts.size(), 1); + EXPECT_TRUE(fake_state->last_request->conjuncts.front()->root()->is_deterministic()); EXPECT_TRUE(predicate_executed); EXPECT_EQ(block.rows(), 0); ASSERT_TRUE(reader.close().ok()); @@ -1846,7 +1863,7 @@ TEST(TableReaderTest, ResidualFilteringHasDedicatedProfileTimer) { ASSERT_TRUE(reader.close().ok()); } -TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { +TEST(TableReaderTest, ConstantPruningSkipsUnsafeSlotlessPredicate) { std::vector projected_columns; auto partition_column = make_table_column(0, "part", std::make_shared()); partition_column.is_partition_key = true; @@ -1876,6 +1893,10 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { .scanner_profile = nullptr, }) .ok()); + // Opening a slotless VExpr caches its constant result. Reset the observation so the assertions + // below distinguish split pruning from normal expression-context initialization. + predicate_executed = false; + SplitReadOptions split; split.current_range.__set_path("fake-table-reader-input"); split.partition_values.emplace("part", Field::create_field(7)); @@ -1884,19 +1905,9 @@ TEST(TableReaderTest, ConstantPruningStopsAtUnsafeSlotlessPredicate) { Block block = build_table_block(projected_columns); bool eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); - EXPECT_TRUE(predicate_executed); - EXPECT_FALSE(eos); - // The later partition predicate is false for part=7. Opening the file proves constant pruning - // stopped at the earlier unsafe expression even though that expression had no slot and thus no - // entry in `_table_filters`. - EXPECT_EQ(fake_state->open_count, 1); - ASSERT_NE(fake_state->last_request, nullptr); - // A slotless unsafe conjunct is an ordering barrier even though it has no TableFilter entry. - // The later predicate must stay on the post-materialization path instead of running inside the - // file reader before the unsafe conjunct. - EXPECT_TRUE(fake_state->last_request->conjuncts.empty()); - ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + EXPECT_FALSE(predicate_executed); EXPECT_TRUE(eos); + EXPECT_EQ(fake_state->open_count, 0); ASSERT_TRUE(reader.close().ok()); } @@ -5362,11 +5373,19 @@ TEST(TableReaderTest, RecomputesPredicateExecutionLayerForEverySplit) { set_name_identifiers(&projected_columns); RuntimeState state {TQueryOptions(), TQueryGlobals()}; - TableReader reader; + std::vector residual_invocations; + auto residual_predicate = std::make_shared(&residual_invocations); + residual_predicate->add_child(table_int32_slot_ref(0, 0, "id")); + SplitLocalPredicateTableReader reader; ASSERT_TRUE(reader.init({ .projected_columns = projected_columns, - .conjuncts = {prepared_conjunct( - &state, table_int32_greater_than_expr(0, 0, 2))}, + .conjuncts = + { + prepared_conjunct( + &state, + table_int32_greater_than_expr(0, 0, 2)), + prepared_conjunct(&state, residual_predicate), + }, .format = FileFormat::PARQUET, .scan_params = nullptr, .io_ctx = nullptr, @@ -5381,17 +5400,21 @@ TEST(TableReaderTest, RecomputesPredicateExecutionLayerForEverySplit) { ASSERT_TRUE(reader.get_block(&block, &eos).ok()); ASSERT_FALSE(eos); expect_int32_column_values(*block.get_by_position(0).column, {3}); + ASSERT_EQ(reader.predicate_layer_counts().size(), 1); + EXPECT_EQ(reader.predicate_layer_counts()[0], (std::pair {1, 1})); ASSERT_TRUE(reader.close().ok()); - // The same predicate cannot be file-local when this split omits `id`. It must be rebuilt as a - // table-level predicate over the materialized NULL instead of inheriting the previous split's - // file-local ownership or escaping without exact evaluation. + // The deterministic predicate was file-local for the previous split while the stateful + // predicate remained residual. When this split omits `id`, both must be rebuilt as table-level + // predicates instead of inheriting the previous split's file-local ownership. ASSERT_TRUE(reader.prepare_split(build_split_options(missing_file)).ok()); block = build_table_block(projected_columns); eos = false; ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_FALSE(eos); EXPECT_EQ(block.rows(), 0); + ASSERT_EQ(reader.predicate_layer_counts().size(), 2); + EXPECT_EQ(reader.predicate_layer_counts()[1], (std::pair {0, 2})); ASSERT_TRUE(reader.get_block(&block, &eos).ok()); EXPECT_TRUE(eos); @@ -5607,6 +5630,39 @@ TEST(TableReaderTest, CreateScanRequestKeepsPayloadWhenAnyReferencingFilterIsRes EXPECT_TRUE(file_request.predicate_only_columns.empty()); } +TEST(TableReaderTest, CreateScanRequestIgnoresResidualOnAnotherPhysicalColumn) { + const auto int_type = std::make_shared(); + std::vector projected_columns = { + make_table_column(0, "filter_key", int_type), + make_table_column(1, "measure", int_type), + }; + projected_columns[0].value_required_after_filter = false; + const std::vector file_schema = { + make_file_column(0, "filter_key", int_type), + make_file_column(1, "measure", int_type), + }; + + TableColumnMapper mapper; + ASSERT_TRUE(mapper.create_mapping(projected_columns, {}, file_schema).ok()); + + TableFilter localized_filter { + .conjunct = VExprContext::create_shared(table_int32_greater_than_expr(0, 0, 1)), + .global_indices = {GlobalIndex(0)}, + }; + TableFilter unrelated_residual_filter { + .conjunct = VExprContext::create_shared(table_int32_greater_than_expr(1, 0, 2)), + .global_indices = {GlobalIndex(1)}, + .can_localize = false, + }; + + FileScanRequest file_request; + ASSERT_TRUE(mapper.create_scan_request({localized_filter, unrelated_residual_filter}, + projected_columns, &file_request) + .ok()); + + EXPECT_EQ(file_request.predicate_only_columns, std::vector({LocalColumnId(0)})); +} + TEST(TableReaderTest, CreateScanRequestUsesColumnNameForByNamePredicateMapping) { const auto int_type = std::make_shared(); std::vector projected_columns = {