Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion be/src/exec/scan/file_scanner_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -632,7 +632,10 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
VExprContextSPtrs conjuncts;
RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts));
VExprContextSPtrs partition_prune_conjuncts;
if (_state->query_options().enable_runtime_filter_partition_prune) {
if (!partition_values.empty()) {
// A split without partition constants cannot be pruned here, so avoid cloning every
// conjunct solely for a consumer that must return immediately. FileScannerV2 otherwise
// keeps safe partition pruning enabled independently of the legacy session gate.
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
}
RETURN_IF_ERROR(_table_reader->prepare_split({
Expand Down
10 changes: 5 additions & 5 deletions be/src/format_v2/parquet/parquet_statistics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -452,11 +452,11 @@ std::optional<format::LocalColumnId> file_column_id_by_block_position(
return std::nullopt;
}

bool has_expr_zonemap_filter(const format::FileScanRequest& request,
const RuntimeState* runtime_state) {
if (!expr_zonemap::is_expr_zonemap_filter_enabled(runtime_state)) {
return false;
}
bool has_expr_zonemap_filter(const format::FileScanRequest& request, const RuntimeState*) {
// FileScannerV2 metadata pruning is a fixed part of its scan pipeline and must not inherit
// the legacy scanner's expression ZoneMap session gate.
// TODO: Fence metadata pruning at the first unsafe/error-preserving conjunct so a later
// ZoneMap predicate cannot bypass its row-level evaluation.
for (const auto& conjunct : request.conjuncts) {
if (conjunct != nullptr && conjunct->root() != nullptr &&
conjunct->root()->can_evaluate_zonemap_filter()) {
Expand Down
60 changes: 60 additions & 0 deletions be/test/exec/scan/file_scanner_v2_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,18 @@ class RetryableCloseTableReader final : public format::TableReader {
std::shared_ptr<RetryableCloseState> _state;
};

class CapturingSplitTableReader final : public format::TableReader {
public:
Status prepare_split(const format::SplitReadOptions& options) override {
conjunct_count = options.conjuncts.has_value() ? options.conjuncts->size() : 0;
partition_prune_conjunct_count = options.partition_prune_conjuncts.size();
return Status::OK();
}

size_t conjunct_count = 0;
size_t partition_prune_conjunct_count = 0;
};

VExprSPtr slot_ref(int slot_id, int column_id, DataTypePtr type, const std::string& name) {
return VSlotRef::create_shared(slot_id, column_id, -1, std::move(type), name);
}
Expand Down Expand Up @@ -478,6 +490,54 @@ TEST(FileScannerV2Test, FailedTableReaderCloseCanBeRetriedThroughScanner) {
EXPECT_EQ(close_state->close_calls, 2);
}

TEST(FileScannerV2Test, PartitionPruningRemainsEnabledWhenSessionSwitchIsFalse) {
TQueryOptions query_options;
query_options.__set_enable_runtime_filter_partition_prune(false);
RuntimeState state {query_options, TQueryGlobals()};
ObjectPool pool;
TDescriptorTable thrift_descriptors;
TTupleDescriptor tuple_descriptor;
tuple_descriptor.id = 0;
tuple_descriptor.byteSize = 0;
tuple_descriptor.numNullBytes = 0;
thrift_descriptors.tupleDescriptors.push_back(tuple_descriptor);
DescriptorTbl* descriptors = nullptr;
ASSERT_TRUE(DescriptorTbl::create(&pool, thrift_descriptors, &descriptors).ok());
TPlanNode plan_node;
plan_node.node_id = 0;
plan_node.node_type = TPlanNodeType::FILE_SCAN_NODE;
plan_node.num_children = 0;
plan_node.limit = -1;
plan_node.row_tuples.push_back(0);
plan_node.file_scan_node.tuple_id = 0;
plan_node.__isset.file_scan_node = true;
FileScanOperatorX parent(&pool, plan_node, 0, *descriptors, 1);
FileScanLocalState local_state(&state, &parent);
RuntimeProfile profile("file_scanner_v2_partition_prune");
auto table_reader = std::make_unique<CapturingSplitTableReader>();
auto* captured = table_reader.get();
FileScannerV2 scanner(&state, &profile, std::move(table_reader));
scanner._local_state = &local_state;

TFileScanRangeParams params;
params.__set_format_type(TFileFormatType::FORMAT_PARQUET);
scanner._params = &params;
scanner._slot_id_to_global_index.emplace(7, format::GlobalIndex(0));
scanner._conjuncts = {VExprContext::create_shared(
slot_ref(7, 7, std::make_shared<DataTypeInt32>(), "partition_col"))};

const auto range = range_with_format("hive", TFileFormatType::FORMAT_PARQUET);
ASSERT_TRUE(scanner._prepare_table_reader_split(range, {}).ok());
EXPECT_EQ(captured->conjunct_count, 1);
EXPECT_EQ(captured->partition_prune_conjunct_count, 0);

ASSERT_TRUE(scanner._prepare_table_reader_split(
range, {{"partition_col", Field::create_field<TYPE_INT>(1)}})
.ok());
EXPECT_EQ(captured->conjunct_count, 1);
EXPECT_EQ(captured->partition_prune_conjunct_count, 1);
}

// Scenario: Once FileScannerV2 is selected, an unsupported range must fail instead of falling back
// to FileScanner.
TEST(FileScannerV2Test, ValidateScanRangeRejectsUnsupportedRange) {
Expand Down
74 changes: 74 additions & 0 deletions be/test/format_v2/parquet/parquet_statistics_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include "format_v2/parquet/parquet_file_context.h"
#include "format_v2/parquet/reader/native/block_split_bloom_filter.h"
#include "io/fs/file_reader.h"
#include "runtime/runtime_state.h"
#include "util/thrift_util.h"
namespace doris {
namespace {
Expand Down Expand Up @@ -669,6 +670,79 @@ TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder)
EXPECT_TRUE(selected_ranges.empty());
}

TEST(NativeParquetStatisticsTest, ZonemapPruningIgnoresDisabledSessionSwitch) {
auto encode_int32 = [](int32_t value) {
std::string bytes(sizeof(value), '\0');
memcpy(bytes.data(), &value, sizeof(value));
return bytes;
};

auto column_schema = std::make_unique<format::parquet::ParquetColumnSchema>();
column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
column_schema->local_id = 0;
column_schema->leaf_column_id = 0;
column_schema->type = std::make_shared<DataTypeInt32>();
column_schema->type_descriptor.doris_type = column_schema->type;
column_schema->type_descriptor.physical_type = tparquet::Type::INT32;
std::vector<std::unique_ptr<format::parquet::ParquetColumnSchema>> schema;
schema.push_back(std::move(column_schema));

tparquet::Statistics statistics;
statistics.__set_min_value(encode_int32(1));
statistics.__set_max_value(encode_int32(2));
statistics.__set_null_count(0);
tparquet::ColumnMetaData column_metadata;
column_metadata.__set_type(tparquet::Type::INT32);
column_metadata.__set_num_values(1);
column_metadata.__set_statistics(statistics);
tparquet::ColumnChunk chunk;
chunk.__set_meta_data(column_metadata);
tparquet::RowGroup row_group;
row_group.__set_columns({chunk});
row_group.__set_num_rows(1);
tparquet::ColumnOrder order;
order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder());
tparquet::FileMetaData metadata;
metadata.__set_column_orders({order});
metadata.__set_row_groups({row_group});

format::FileScanRequest request;
request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0));
request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))};
request.conjuncts = {
VExprContext::create_shared(std::make_shared<MetadataInt32GreaterThanExpr>(100))};

TQueryOptions query_options;
query_options.__set_enable_expr_zonemap_filter(false);
RuntimeState state {query_options, TQueryGlobals()};
std::vector<int> selected_row_groups;
ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr,
&selected_row_groups, false, nullptr,
nullptr, &state)
.ok());
EXPECT_TRUE(selected_row_groups.empty());

format::parquet::NativeParquetPageIndex page_index;
page_index.column_index.__set_min_values({encode_int32(1)});
page_index.column_index.__set_max_values({encode_int32(2)});
page_index.column_index.__set_null_pages({false});
page_index.column_index.__set_null_counts({0});
tparquet::PageLocation location;
location.__set_offset(0);
location.__set_compressed_page_size(10);
location.__set_first_row_index(0);
page_index.offset_index.__set_page_locations({location});
std::unordered_map<int, format::parquet::NativeParquetPageIndex> page_indexes;
page_indexes.emplace(0, std::move(page_index));
std::vector<format::parquet::RowRange> selected_ranges;
std::map<int, format::parquet::ParquetPageSkipPlan> skip_plans;
ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index(
metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans,
nullptr, nullptr, &state)
.ok());
EXPECT_TRUE(selected_ranges.empty());
}

TEST(NativeParquetStatisticsTest, ContradictoryAllNullPageCountsDisablePruning) {
auto column_schema = std::make_unique<format::parquet::ParquetColumnSchema>();
column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE;
Expand Down
16 changes: 13 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
Original file line number Diff line number Diff line change
Expand Up @@ -2198,7 +2198,15 @@ public boolean isEnableHboNonStrictMatchingMode() {
@VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_PRUNE, needForward = true, fuzzy = true)
public boolean enableRuntimeFilterPrune = true;

@VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_PARTITION_PRUNE, needForward = true, fuzzy = true)
@VarAttrDef.VarAttr(
name = ENABLE_RUNTIME_FILTER_PARTITION_PRUNE,
description = {"控制支持该变量的 scanner 是否启用运行时过滤器分区裁剪。"
+ "File Scanner V2 始终启用安全的分区裁剪。默认为 true。",
"Controls runtime-filter partition pruning in scanners that honor this variable. "
+ "File Scanner V2 always enables safe partition pruning. "
+ "The default value is true."},
needForward = true,
fuzzy = true)
public boolean enableRuntimeFilterPartitionPrune = true;

/**
Expand Down Expand Up @@ -2719,8 +2727,10 @@ public static boolean isEagerAggregationOnJoin() {
@VarAttrDef.VarAttr(
name = ENABLE_EXPR_ZONEMAP_FILTER,
fuzzy = true,
description = {"控制 scanner 是否启用表达式 ZoneMap 过滤。默认为 true。",
"Controls whether to enable expression ZoneMap filtering in scanners. "
description = {"控制支持该变量的 scanner 是否启用表达式 ZoneMap 过滤。"
+ "File Scanner V2 始终启用安全的表达式 ZoneMap 过滤。默认为 true。",
"Controls expression ZoneMap filtering in scanners that honor this variable. "
+ "File Scanner V2 always enables safe expression ZoneMap filtering. "
+ "The default value is true."},
needForward = true)
public boolean enableExprZonemapFilter = true;
Expand Down
5 changes: 4 additions & 1 deletion gensrc/thrift/PaloInternalService.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,8 @@ struct TQueryOptions {

148: optional i32 min_scanners_concurrency = 1;
149: optional i32 min_scan_scheduler_concurrency = 0; //deprecated
// Controls runtime-filter partition pruning for readers that honor this option.
// FileScannerV2 always enables safe partition pruning.
150: optional bool enable_runtime_filter_partition_prune = true;

// The minimum memory that an operator required to run.
Expand Down Expand Up @@ -501,7 +503,8 @@ struct TQueryOptions {
// enable plan local exchange node in fe
223: optional bool enable_local_shuffle_planner;

// To control whether BE scan readers may apply expression-based ZoneMap pruning.
// Controls expression-based ZoneMap pruning for readers that honor this option.
// FileScannerV2 always enables safe expression ZoneMap pruning.
224: optional bool enable_expr_zonemap_filter = true

225: optional i64 runtime_filter_tree_publish_max_send_bytes = 268435456
Expand Down
Loading