diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetListLayoutResolver.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetListLayoutResolver.java new file mode 100644 index 000000000000..10d44ced537c --- /dev/null +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetListLayoutResolver.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet; + +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.Type; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.format.parquet.ParquetSchemaConverter.LIST_ELEMENT_NAME; +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** + * Resolves Parquet list layouts, following the backward-compatibility rules in the Parquet spec: LogicalTypes#Backward-compatibility-rules. + * + *

All list layout decisions should be made through this class so that schema inference, + * requested-schema clipping and reader construction share a single interpretation. + */ +public final class ParquetListLayoutResolver { + + private static final String LIST_WRAPPER_NAME = "list"; + private static final String LEGACY_LIST_ARRAY_NAME = "array"; + + private ParquetListLayoutResolver() {} + + /** Returns true if the given group is annotated as a Parquet LIST logical type. */ + public static boolean isList(GroupType listType) { + return listType.getLogicalTypeAnnotation() + instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation; + } + + /** + * Whether the given group has the legacy nested-list shape: an unannotated {@code REPEATED} + * group whose single child is also {@code REPEATED}. + * + *

This is the Parquet spec's backward-compatibility Rule 3 and appears as the element + * of an annotated list, e.g. {@code repeated group element { repeated int32 array; }}. The + * repeated child is the element of the nested list. + */ + public static boolean isLegacyNestedList(GroupType groupType) { + return groupType.isRepetition(Type.Repetition.REPEATED) + && groupType.getFieldCount() == 1 + && groupType.getType(0).getRepetition() == Type.Repetition.REPEATED; + } + + /** + * Returns true if the given group is a three-level Parquet list. + * + *

In a three-level list the immediate repeated child is a wrapper group whose single + * non-repeated child is the actual element type. This covers the canonical layout ({@code list + * -> element}) as well as legacy wrappers such as Hive's {@code bag} layout. + * + *

This corresponds to the Parquet spec's backward-compatibility Rule 5: a repeated + * group that contains exactly one non-repeated child is a wrapper, unless it matches one of + * Rules 1-4. + * + *

The compatibility encodings that are not three-level are: + * + *

+ */ + public static boolean isThreeLevelList(Type type) { + if (type.isPrimitive()) { + return false; + } + + GroupType listType = type.asGroupType(); + if (!isList(listType)) { + return false; + } + + // A list must have exactly one repeated child (the middle level). + if (listType.getFieldCount() != 1) { + return false; + } + Type middle = listType.getType(0); + if (middle.isPrimitive() || middle.getRepetition() != Type.Repetition.REPEATED) { + return false; + } + GroupType repeatedGroup = middle.asGroupType(); + + // Rule 5: the repeated group is a wrapper containing exactly one non-repeated child. + if (repeatedGroup.getFieldCount() != 1 + || repeatedGroup.getType(0).getRepetition() == Type.Repetition.REPEATED) { + return false; + } + + // Rule 4: legacy "array" and "_tuple" encodings are not wrappers; the repeated + // group itself is the element type. + return !LEGACY_LIST_ARRAY_NAME.equals(repeatedGroup.getName()) + && !(listType.getName() + "_tuple").equals(repeatedGroup.getName()); + } + + /** + * Returns true if the given group follows the canonical three-level Parquet list layout ({@code + * list -> element}). + * + *

The canonical layout is described in the Parquet spec: LogicalTypes#Lists + */ + public static boolean isCanonicalList(Type type) { + if (!isThreeLevelList(type)) { + return false; + } + + Type middle = type.asGroupType().getType(0); + Type element = middle.asGroupType().getType(0); + return LIST_WRAPPER_NAME.equals(middle.getName()) + && LIST_ELEMENT_NAME.equals(element.getName()); + } + + /** + * Resolves the element type of the given LIST-annotated group according to the Parquet spec's + * backward-compatibility rules for lists. + * + *

For a three-level list (Rule 5) the returned type is the single child of the repeated + * wrapper. For Rules 1-4 the repeated field itself is returned because it is the element type. + */ + public static Type resolveElementType(GroupType listType) { + checkArgument( + isList(listType) || isLegacyNestedList(listType), + "Expected LIST-annotated group but got: %s", + listType); + + if (isThreeLevelList(listType)) { + return listType.getType(0).asGroupType().getType(0); + } + + return listType.getType(0); + } + + /** + * A schema-level manifest of list layouts, modeled after parquet-cpp's {@code SchemaManifest}. + * + *

Built once from the file schema, it resolves the three-level verdict of every + * LIST-annotated node in the file and records it by field path, so that reader construction + * interprets list layouts against the file schema rather than the (possibly reshaped) requested + * schema. Field paths are the identifier shared by the file schema, the requested schema and + * the ColumnIO tree: clipping may rebuild nodes, but it preserves names, so paths stay stable + * where node identity would not. + * + *

Paths absent from this context belong to nodes that are not LIST-annotated in the file + * schema (or synthetic fill fields, whose requested shape is Paimon-canonical); the verdict + * falls back to interpreting the requested node itself. + */ + public static final class LayoutContext { + private final Map, Boolean> threeLevelMapping = new HashMap<>(); + + /** Builds the manifest by resolving every LIST-annotated node of the file schema. */ + public static LayoutContext fromFileSchema(GroupType fileSchema) { + LayoutContext context = new LayoutContext(); + for (Type field : fileSchema.getFields()) { + collect(field, Collections.singletonList(field.getName()), context); + } + return context; + } + + private static void collect(Type node, List path, LayoutContext context) { + if (node.isPrimitive()) { + return; + } + GroupType group = node.asGroupType(); + if (isList(group)) { + context.threeLevelMapping.put( + path, ParquetListLayoutResolver.isThreeLevelList(group)); + } + for (Type child : group.getFields()) { + List childPath = new ArrayList<>(path); + childPath.add(child.getName()); + collect(child, childPath, context); + } + } + + /** + * Returns whether the list at {@code path} is a three-level list, as resolved against the + * file schema; falls back to interpreting {@code requestedGroup} itself when the path is + * not an annotated list in the file schema. + */ + public boolean isThreeLevelList(GroupType requestedGroup, String[] path) { + Boolean threeLevel = threeLevelMapping.get(Arrays.asList(path)); + return threeLevel != null + ? threeLevel + : ParquetListLayoutResolver.isThreeLevelList(requestedGroup); + } + } +} diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java index ab7d277a6221..c4719a3cb153 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetReaderFactory.java @@ -26,6 +26,7 @@ import org.apache.paimon.data.shredding.ShreddingReadPlan; import org.apache.paimon.format.FormatMetadataUtils; import org.apache.paimon.format.FormatReaderFactory; +import org.apache.paimon.format.parquet.ParquetListLayoutResolver.LayoutContext; import org.apache.paimon.format.parquet.reader.VectorizedParquetRecordReader; import org.apache.paimon.format.parquet.type.ParquetField; import org.apache.paimon.format.shredding.ShreddingFormatReader; @@ -56,7 +57,6 @@ import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.OriginalType; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; import org.apache.parquet.schema.Types; @@ -76,8 +76,11 @@ import java.util.function.IntFunction; import static org.apache.paimon.data.columnar.ColumnVectorUtils.createParquetWritableColumnVector; +import static org.apache.paimon.format.parquet.ParquetListLayoutResolver.isLegacyNestedList; +import static org.apache.paimon.format.parquet.ParquetListLayoutResolver.isList; +import static org.apache.paimon.format.parquet.ParquetListLayoutResolver.isThreeLevelList; +import static org.apache.paimon.format.parquet.ParquetListLayoutResolver.resolveElementType; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.PAIMON_SCHEMA; -import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetMapKeyValueType; import static org.apache.paimon.format.parquet.reader.ParquetReaderUtil.buildFieldsList; @@ -239,9 +242,10 @@ private static DataField[] readFields(RowType readType) { } private RequestedSchema createRequestedSchema(MessageType fileSchema, DataField[] readFields) { + LayoutContext listLayout = LayoutContext.fromFileSchema(fileSchema); MessageType rs = clipParquetSchema(fileSchema, readFields); MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(rs); - List f = buildFieldsList(readFields, columnIO, rs); + List f = buildFieldsList(readFields, columnIO, rs, listLayout); return new RequestedSchema(rs, f); } @@ -343,31 +347,37 @@ private Type clipParquetType(DataType readType, Type parquetType) { Preconditions.checkArgument( listSubFields == 1, "Parquet list group type should only have one middle level REPEATED field."); - // There are two representations for array type in parquet. - // See link: - // https://impala.apache.org/docs/build/html/topics/impala_parquet_array_resolution.html. - int level = arrayGroup.getType(0) instanceof GroupType ? 3 : 2; - Type elementType = - clipParquetType(elementReadType, parquetListElementType(arrayGroup)); - - if (level == 3) { - // In case that the name in middle level is not "list". - Type groupMiddle = - new GroupType( - Type.Repetition.REPEATED, - arrayGroup.getType(0).getName(), - elementType); - return new GroupType( - arrayGroup.getRepetition(), - arrayGroup.getName(), - OriginalType.LIST, - groupMiddle); + if (isList(arrayGroup)) { + boolean threeLevel = isThreeLevelList(arrayGroup); + Type originalElement = resolveElementType(arrayGroup); + Type elementType = clipParquetType(elementReadType, originalElement); + if (threeLevel) { + Type clippedMiddle = + arrayGroup + .getType(0) + .asGroupType() + .withNewFields(Collections.singletonList(elementType)); + return arrayGroup.withNewFields(Collections.singletonList(clippedMiddle)); + } else { + // Rules 1-4: the repeated field itself is the element. Keep it (clipped) + // in place so that reader construction unwraps by path against the file + // schema instead of guessing from this reshaped requested shape. + return arrayGroup.withNewFields(Collections.singletonList(elementType)); + } + } else if (isLegacyNestedList(arrayGroup)) { + // Rule 3: an unannotated repeated group is itself the element of the annotated + // outer list, and its single repeated child is the element of the nested inner + // list. + Type originalElement = arrayGroup.getType(0); + Type elementType = clipParquetType(elementReadType, originalElement); + return arrayGroup.withNewFields(Collections.singletonList(elementType)); } else { - return new GroupType( - arrayGroup.getRepetition(), - arrayGroup.getName(), - OriginalType.LIST, - elementType); + throw new IllegalArgumentException( + String.format( + "Cannot read Parquet group '%s' as an ARRAY: it is neither " + + "LIST-annotated nor a legacy nested list. Parquet type: %s, " + + "read type: %s", + arrayGroup.getName(), arrayGroup, readType)); } default: return parquetType; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java index 912baec0721c..8b47c1b5a5c8 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSchemaConverter.java @@ -462,10 +462,18 @@ public static DataField convertToPaimonField(Type parquetType) { parquetType.getId().intValue(), parquetType.getName(), paimonDataType); } else { GroupType groupType = parquetType.asGroupType(); - if (logicalType instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) { + if (ParquetListLayoutResolver.isList(groupType)) { + Type parquetElementType = ParquetListLayoutResolver.resolveElementType(groupType); + DataType elementDataType = convertToPaimonField(parquetElementType).type(); + if (!ParquetListLayoutResolver.isThreeLevelList(groupType)) { + // Rules 1-4: the repeated node itself is the element. A REPEATED node is + // never null, so the element type is not nullable. + elementDataType = elementDataType.notNull(); + } + paimonDataType = new ArrayType(elementDataType); + } else if (ParquetListLayoutResolver.isLegacyNestedList(groupType)) { paimonDataType = - new ArrayType( - convertToPaimonField(parquetListElementType(groupType)).type()); + new ArrayType(convertToPaimonField(groupType.getType(0)).type().notNull()); } else if (logicalType instanceof LogicalTypeAnnotation.MapLogicalTypeAnnotation) { Pair keyValueType = parquetMapKeyValueType(groupType); paimonDataType = @@ -490,22 +498,6 @@ public static DataField convertToPaimonField(Type parquetType) { return new DataField(parquetType.getId().intValue(), parquetType.getName(), paimonDataType); } - public static Type parquetListElementType(GroupType listType) { - int level = listType.getType(0) instanceof GroupType ? 3 : 2; - if (level == 3) { - // Level 3 representation of list type. - // List type should only have one middle group type, which is repeated, and one element - // type, which is optional. - return listType.getType(0).asGroupType().getType(0); - } else if (level == 2) { - // Level 2 representation of list type - return listType.getType(0); - } else { - throw new UnsupportedOperationException( - "Parquet list type only have two level representation and three level representation."); - } - } - public static Pair parquetMapKeyValueType(GroupType mapType) { GroupType keyValue = mapType.getType(0).asGroupType(); return Pair.of(keyValue.getType(MAP_KEY_NAME), keyValue.getType(MAP_VALUE_NAME)); diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java index 0e902c56e307..3d33f6c7d3ff 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingReadPlanFactory.java @@ -56,7 +56,7 @@ import static org.apache.paimon.data.variant.Variant.METADATA; import static org.apache.paimon.data.variant.Variant.VALUE; -import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType; +import static org.apache.paimon.format.parquet.ParquetListLayoutResolver.resolveElementType; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetMapKeyValueType; /** @@ -194,7 +194,7 @@ private static DataType buildPhysicalType( arrayType.isNullable(), buildPhysicalType( arrayType.getElementType(), - parquetListElementType(fileType.asGroupType()), + resolveElementType(fileType.asGroupType()), caseSensitive)); case VECTOR: VectorType vectorType = (VectorType) logicalType; @@ -203,7 +203,7 @@ private static DataType buildPhysicalType( vectorType.getLength(), buildPhysicalType( vectorType.getElementType(), - parquetListElementType(fileType.asGroupType()), + resolveElementType(fileType.asGroupType()), caseSensitive)); case MAP: MapType mapType = (MapType) logicalType; diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java index f9e5075342e1..860c5df9d95a 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/VariantShreddingTypePruner.java @@ -25,7 +25,6 @@ import org.apache.paimon.types.RowType; import org.apache.parquet.schema.GroupType; -import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.Type; import javax.annotation.Nullable; @@ -38,7 +37,8 @@ import java.util.Map; import java.util.Set; -import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType; +import static org.apache.paimon.format.parquet.ParquetListLayoutResolver.isCanonicalList; +import static org.apache.paimon.format.parquet.ParquetListLayoutResolver.resolveElementType; import static org.apache.paimon.utils.Preconditions.checkArgument; /** @@ -53,9 +53,6 @@ * objectSchemaMap}. */ public class VariantShreddingTypePruner { - private static final String LIST_WRAPPER_NAME = "list"; - private static final String LIST_ELEMENT_NAME = "element"; - @Nullable private final PathNode root; VariantShreddingTypePruner(RowType variantRowType) { @@ -197,7 +194,7 @@ private GroupType clipListShreddingRow(GroupType group, PathNode node, ListThe canonical layout is described in the Parquet spec: LogicalTypes#Lists - */ - private static boolean isCanonicalList(Type type) { - if (type.isPrimitive()) { - return false; - } - - GroupType listGroup = type.asGroupType(); - // 1. Must be a LIST logical type. - if (!(listGroup.getLogicalTypeAnnotation() - instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation)) { - return false; - } - - // 2. LIST group must have exactly one child named "list". - if (listGroup.getFieldCount() != 1) { - return false; - } - Type middle = listGroup.getType(0); - if (!LIST_WRAPPER_NAME.equals(middle.getName())) { - return false; - } - - // 3. The child must be a repeated group. - if (middle.isPrimitive() || middle.getRepetition() != Type.Repetition.REPEATED) { - return false; - } - GroupType repeatedWrapper = middle.asGroupType(); - - // 4. The repeated wrapper must contain exactly one child named "element". - if (repeatedWrapper.getFieldCount() != 1) { - return false; - } - - Type element = repeatedWrapper.getType(0); - return LIST_ELEMENT_NAME.equals(element.getName()); - } - /** Returns true if the given group is a plain struct (not a Parquet list or map). */ private static boolean isObjectGroup(Type type) { if (type.isPrimitive()) { diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java index 48f98b785bd7..92d8ff839a12 100644 --- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java +++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/reader/ParquetReaderUtil.java @@ -18,6 +18,7 @@ package org.apache.paimon.format.parquet.reader; +import org.apache.paimon.format.parquet.ParquetListLayoutResolver; import org.apache.paimon.format.parquet.type.ParquetField; import org.apache.paimon.format.parquet.type.ParquetGroupField; import org.apache.paimon.format.parquet.type.ParquetPrimitiveField; @@ -30,7 +31,6 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.types.VectorType; import org.apache.paimon.utils.Pair; -import org.apache.paimon.utils.StringUtils; import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; @@ -47,34 +47,38 @@ import java.util.stream.Collectors; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToPaimonField; -import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetListElementType; import static org.apache.paimon.format.parquet.ParquetSchemaConverter.parquetMapKeyValueType; -import static org.apache.parquet.schema.Type.Repetition.REPEATED; import static org.apache.parquet.schema.Type.Repetition.REQUIRED; /** Util for generating parquet readers. */ public class ParquetReaderUtil { public static List buildFieldsList( - DataField[] readFields, MessageColumnIO columnIO, MessageType requestedFileSchema) { + DataField[] readFields, + MessageColumnIO columnIO, + MessageType requestedSchema, + ParquetListLayoutResolver.LayoutContext listLayout) { List list = new ArrayList<>(); for (int i = 0; i < readFields.length; i++) { list.add( constructField( readFields[i], lookupColumnByName(columnIO, readFields[i].name()), - requestedFileSchema.getType(i))); + requestedSchema.getType(i), + listLayout)); } return list; } private static ParquetField constructField( - DataField dataField, ColumnIO columnIO, Type parquetType) { + DataField dataField, + ColumnIO columnIO, + Type parquetType, + ParquetListLayoutResolver.LayoutContext listLayout) { boolean required = columnIO.getType().getRepetition() == REQUIRED; int repetitionLevel = columnIO.getRepetitionLevel(); int definitionLevel = columnIO.getDefinitionLevel(); DataType type = dataField.type(); - String fieldName = dataField.name(); if (type instanceof RowType) { GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; RowType rowType = (RowType) type; @@ -88,7 +92,8 @@ private static ParquetField constructField( constructField( children.get(i), lookupColumnByName(groupColumnIO, childName), - getTypeIgnoreCase(parquetType.asGroupType(), childName))); + getTypeIgnoreCase(parquetType.asGroupType(), childName), + listLayout)); } GroupType parquetGroup = parquetType.asGroupType(); for (int i = children.size(); i < parquetGroup.getFieldCount(); i++) { @@ -98,7 +103,8 @@ private static ParquetField constructField( constructField( extraField, lookupColumnByName(groupColumnIO, extraType.getName()), - extraType)); + extraType, + listLayout)); } return new ParquetGroupField( @@ -119,12 +125,14 @@ private static ParquetField constructField( constructField( new DataField(0, "", mapType.getKeyType()), keyValueColumnIO.getChild(0), - keyValueType.getKey()); + keyValueType.getKey(), + listLayout); ParquetField valueField = constructField( new DataField(0, "", mapType.getValueType()), keyValueColumnIO.getChild(1), - keyValueType.getValue()); + keyValueType.getValue(), + listLayout); return new ParquetGroupField( type, repetitionLevel, @@ -143,12 +151,14 @@ private static ParquetField constructField( constructField( new DataField(0, "", multisetType.getElementType()), keyValueColumnIO.getChild(0), - keyValueType.getKey()); + keyValueType.getKey(), + listLayout); ParquetField valueField = constructField( new DataField(0, "", new IntType()), keyValueColumnIO.getChild(1), - keyValueType.getValue()); + keyValueType.getValue(), + listLayout); return new ParquetGroupField( type, repetitionLevel, @@ -163,34 +173,26 @@ private static ParquetField constructField( type instanceof ArrayType ? ((ArrayType) type).getElementType() : ((VectorType) type).getElementType(); - ColumnIO elementTypeColumnIO; - if (columnIO instanceof GroupColumnIO) { - GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; - if (!StringUtils.isNullOrWhitespaceOnly(fieldName)) { - // Column lookup is case-insensitive; the wrapper-group names can - // therefore differ in case from the requested field name. - while (!groupColumnIO.getName().equalsIgnoreCase(fieldName)) { - groupColumnIO = (GroupColumnIO) groupColumnIO.getChild(0); - } - elementTypeColumnIO = groupColumnIO; - } else { - if (elementType instanceof RowType) { - elementTypeColumnIO = groupColumnIO; - } else { - elementTypeColumnIO = groupColumnIO.getChild(0); - } - } - } else if (columnIO instanceof PrimitiveColumnIO) { - elementTypeColumnIO = columnIO; - } else { - throw new RuntimeException(String.format("Unknown ColumnIO, %s", columnIO)); - } + GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO; + GroupType requestedGroup = parquetType.asGroupType(); + + boolean threeLevel = + listLayout.isThreeLevelList(requestedGroup, groupColumnIO.getFieldPath()); + Type requestedElementType = + threeLevel + ? requestedGroup.getType(0).asGroupType().getType(0) + : requestedGroup.getType(0); + + ColumnIO middleColumnIO = groupColumnIO.getChild(0); + ColumnIO elementColumnIO = + threeLevel ? ((GroupColumnIO) middleColumnIO).getChild(0) : middleColumnIO; ParquetField field = constructField( new DataField(0, "", elementType), - getArrayElementColumn(elementTypeColumnIO), - parquetListElementType(parquetType.asGroupType())); + elementColumnIO, + requestedElementType, + listLayout); if (repetitionLevel == field.getRepetitionLevel()) { repetitionLevel = columnIO.getParent().getRepetitionLevel(); } @@ -270,32 +272,4 @@ public static GroupColumnIO getMapKeyValueColumn(GroupColumnIO groupColumnIO) { } return groupColumnIO; } - - public static ColumnIO getArrayElementColumn(ColumnIO columnIO) { - while (columnIO instanceof GroupColumnIO && !columnIO.getType().isRepetition(REPEATED)) { - columnIO = ((GroupColumnIO) columnIO).getChild(0); - } - - /* Compatible with array has a standard 3-level structure: - * optional group my_list (LIST) { - * repeated group element { - * required binary str (UTF8); - * }; - * } - */ - if (columnIO instanceof GroupColumnIO - && columnIO.getType().getLogicalTypeAnnotation() == null - && ((GroupColumnIO) columnIO).getChildrenCount() == 1 - && !columnIO.getName().equals("array") - && !columnIO.getName().equals(columnIO.getParent().getName() + "_tuple")) { - return ((GroupColumnIO) columnIO).getChild(0); - } - - /* Compatible with array for 2-level arrays where a repeated field is not a group: - * optional group my_list (LIST) { - * repeated int32 element; - * } - */ - return columnIO; - } } diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetLegacyListReadTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetLegacyListReadTest.java new file mode 100644 index 000000000000..2cdff426b1af --- /dev/null +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetLegacyListReadTest.java @@ -0,0 +1,534 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.format.FormatReaderContext; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; + +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.util.HadoopOutputFile; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * End-to-end tests for reading Parquet files that use legacy list encodings, as described by the + * backward-compatibility rules in the Parquet spec. + */ +public class ParquetLegacyListReadTest { + + private static final int ROW_COUNT = 10; + + @TempDir public File folder; + + /** Backward-compatibility Rule 1: a repeated primitive field is itself the element type. */ + @Test + public void testReadRepeatedPrimitiveElementList() throws Exception { + // optional group my_list (LIST) { repeated int32 element; } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .repeated(INT32) + .named("element") + .named("my_list")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + Group array = row.addGroup("my_list"); + int size = i % 3 + 1; + for (int j = 0; j < size; j++) { + array.append("element", i * 10 + j); + } + writer.write(row); + } + } + + AtomicInteger i = new AtomicInteger(); + try (RecordReader reader = + createReader( + path, + new RowType( + Collections.singletonList( + new DataField( + 0, "my_list", new ArrayType(new IntType())))))) { + reader.forEachRemaining( + row -> { + int index = i.getAndIncrement(); + InternalArray array = row.getArray(0); + int size = index % 3 + 1; + assertThat(array.size()).isEqualTo(size); + for (int j = 0; j < size; j++) { + assertThat(array.getInt(j)).isEqualTo(index * 10 + j); + } + }); + } + assertThat(i.get()).isEqualTo(ROW_COUNT); + } + + /** + * Backward-compatibility Rule 2: a repeated group with multiple fields is itself the element + * type. + */ + @Test + public void testReadRepeatedStructElementList() throws Exception { + // optional group my_list (LIST) { repeated group element { optional int32 x; optional int32 + // y; } } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("x") + .optional(INT32) + .named("y") + .named("element")) + .named("my_list")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + Group array = row.addGroup("my_list"); + int size = i % 3 + 1; + for (int j = 0; j < size; j++) { + array.addGroup("element").append("x", i + j).append("y", i * j); + } + writer.write(row); + } + } + + RowType elementType = + new RowType( + Arrays.asList( + new DataField(0, "x", new IntType()), + new DataField(1, "y", new IntType()))); + AtomicInteger i = new AtomicInteger(); + try (RecordReader reader = + createReader( + path, + new RowType( + Collections.singletonList( + new DataField( + 0, "my_list", new ArrayType(elementType)))))) { + reader.forEachRemaining( + row -> { + int index = i.getAndIncrement(); + InternalArray array = row.getArray(0); + int size = index % 3 + 1; + assertThat(array.size()).isEqualTo(size); + for (int j = 0; j < size; j++) { + InternalRow element = array.getRow(j, 2); + assertThat(element.getInt(0)).isEqualTo(index + j); + assertThat(element.getInt(1)).isEqualTo(index * j); + } + }); + } + assertThat(i.get()).isEqualTo(ROW_COUNT); + } + + /** + * Backward-compatibility Rule 2 with projection: a repeated group with multiple fields is the + * element type, and projecting it to a single field must not reclassify the clipped element + * group as a Rule 5 wrapper. + */ + @Test + public void testReadStructElementWithProjection() throws Exception { + // optional group my_list (LIST) { repeated group element { optional int32 x; optional int32 + // y; } } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("x") + .optional(INT32) + .named("y") + .named("element")) + .named("my_list")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + Group array = row.addGroup("my_list"); + int size = i % 3 + 1; + for (int j = 0; j < size; j++) { + array.addGroup("element").append("x", i + j).append("y", i * j); + } + writer.write(row); + } + } + + RowType elementType = + new RowType(Collections.singletonList(new DataField(0, "x", new IntType()))); + AtomicInteger i = new AtomicInteger(); + try (RecordReader reader = + createReader( + path, + new RowType( + Collections.singletonList( + new DataField( + 0, "my_list", new ArrayType(elementType)))))) { + reader.forEachRemaining( + row -> { + int index = i.getAndIncrement(); + InternalArray array = row.getArray(0); + int size = index % 3 + 1; + assertThat(array.size()).isEqualTo(size); + for (int j = 0; j < size; j++) { + assertThat(array.getRow(j, 1).getInt(0)).isEqualTo(index + j); + } + }); + } + assertThat(i.get()).isEqualTo(ROW_COUNT); + } + + /** + * Backward-compatibility Rule 3: a repeated group whose single child is also repeated is the + * element type, and the repeated child is a nested list. Reading {@code [[8, 9], ...]} must + * return every inner value, not just the first one. + */ + @Test + public void testReadNestedLegacyList() throws Exception { + // optional group my_list (LIST) { repeated group element { repeated int32 array; } } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .repeated(INT32) + .named("array") + .named("array")) + .named("my_list")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + Group array = row.addGroup("my_list"); + int size = i % 3 + 1; + for (int j = 0; j < size; j++) { + Group element = array.addGroup("array"); + element.append("array", i * 10 + j); + element.append("array", i * 10 + j + 100); + } + writer.write(row); + } + } + + AtomicInteger i = new AtomicInteger(); + try (RecordReader reader = + createReader( + path, + new RowType( + Collections.singletonList( + new DataField( + 0, + "my_list", + new ArrayType(new ArrayType(new IntType()))))))) { + reader.forEachRemaining( + row -> { + int index = i.getAndIncrement(); + InternalArray array = row.getArray(0); + int size = index % 3 + 1; + assertThat(array.size()).isEqualTo(size); + for (int j = 0; j < size; j++) { + InternalArray nested = array.getArray(j); + assertThat(nested.size()).isEqualTo(2); + assertThat(nested.getInt(0)).isEqualTo(index * 10 + j); + assertThat(nested.getInt(1)).isEqualTo(index * 10 + j + 100); + } + }); + } + assertThat(i.get()).isEqualTo(ROW_COUNT); + } + + /** + * Backward-compatibility Rule 4: a repeated group named {@code "array"} with one field is the + * element type. + */ + @Test + public void testReadArrayNamedGroupElementList() throws Exception { + // optional group my_list (LIST) { repeated group array { optional int32 foo; } } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("foo") + .named("array")) + .named("my_list")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + Group array = row.addGroup("my_list"); + int size = i % 3 + 1; + for (int j = 0; j < size; j++) { + array.addGroup("array").append("foo", i * 10 + j); + } + writer.write(row); + } + } + + RowType elementType = + new RowType(Collections.singletonList(new DataField(0, "foo", new IntType()))); + AtomicInteger i = new AtomicInteger(); + try (RecordReader reader = + createReader( + path, + new RowType( + Collections.singletonList( + new DataField( + 0, "my_list", new ArrayType(elementType)))))) { + reader.forEachRemaining( + row -> { + int index = i.getAndIncrement(); + InternalArray array = row.getArray(0); + int size = index % 3 + 1; + assertThat(array.size()).isEqualTo(size); + for (int j = 0; j < size; j++) { + assertThat(array.getRow(j, 1).getInt(0)).isEqualTo(index * 10 + j); + } + }); + } + assertThat(i.get()).isEqualTo(ROW_COUNT); + } + + /** + * Backward-compatibility Rule 4: a repeated group named {@code "_tuple"} with one field + * is the element type. + */ + @Test + public void testReadListTupleNamedGroupElementList() throws Exception { + // optional group my_list (LIST) { repeated group my_list_tuple { required binary str + // (STRING); } } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .required(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("str") + .named("my_list_tuple")) + .named("my_list")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + Group array = row.addGroup("my_list"); + int size = i % 3 + 1; + for (int j = 0; j < size; j++) { + array.addGroup("my_list_tuple") + .append("str", Binary.fromString("v" + (i * 10 + j))); + } + writer.write(row); + } + } + + RowType elementType = + new RowType( + Collections.singletonList( + new DataField(0, "str", new VarCharType(VarCharType.MAX_LENGTH)))); + AtomicInteger i = new AtomicInteger(); + try (RecordReader reader = + createReader( + path, + new RowType( + Collections.singletonList( + new DataField( + 0, "my_list", new ArrayType(elementType)))))) { + reader.forEachRemaining( + row -> { + int index = i.getAndIncrement(); + InternalArray array = row.getArray(0); + int size = index % 3 + 1; + assertThat(array.size()).isEqualTo(size); + for (int j = 0; j < size; j++) { + assertThat(array.getRow(j, 1).getString(0)) + .isEqualTo(BinaryString.fromString("v" + (index * 10 + j))); + } + }); + } + assertThat(i.get()).isEqualTo(ROW_COUNT); + } + + /** + * Backward-compatibility Rule 5: a repeated group with a single non-repeated child is a + * wrapper, so the child is the element (Hive bag style). + */ + @Test + public void testReadElementWrappedList() throws Exception { + // optional group my_list (LIST) { repeated group element { optional int32 num; } } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("num") + .named("element")) + .named("my_list")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + Group array = row.addGroup("my_list"); + int size = i % 3 + 1; + for (int j = 0; j < size; j++) { + array.addGroup("element").append("num", i * 10 + j); + } + writer.write(row); + } + } + + AtomicInteger i = new AtomicInteger(); + try (RecordReader reader = + createReader( + path, + new RowType( + Collections.singletonList( + new DataField( + 0, "my_list", new ArrayType(new IntType())))))) { + reader.forEachRemaining( + row -> { + int index = i.getAndIncrement(); + InternalArray array = row.getArray(0); + int size = index % 3 + 1; + assertThat(array.size()).isEqualTo(size); + for (int j = 0; j < size; j++) { + assertThat(array.getInt(j)).isEqualTo(index * 10 + j); + } + }); + } + assertThat(i.get()).isEqualTo(ROW_COUNT); + } + + /** + * A plain struct group is neither LIST-annotated nor a legacy nested list, so reading it as an + * ARRAY must fail fast in schema clipping instead of silently producing undefined data. + */ + @Test + public void testReadNonListGroupAsArrayFails() throws Exception { + // optional group arr { optional int32 x; optional int32 y; } + MessageType schema = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .addField( + Types.buildGroup(Type.Repetition.OPTIONAL) + .optional(INT32) + .named("x") + .optional(INT32) + .named("y") + .named("tuple")) + .named("arr")); + Path path = new Path(folder.getPath(), UUID.randomUUID().toString()); + try (ParquetWriter writer = createWriter(path, schema)) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < ROW_COUNT; i++) { + Group row = factory.newGroup(); + row.addGroup("arr").addGroup("tuple").append("x", i).append("y", i * 2); + writer.write(row); + } + } + + RowType readType = + new RowType( + Collections.singletonList( + new DataField(0, "arr", new ArrayType(new IntType())))); + assertThatThrownBy(() -> createReader(path, readType)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet group 'arr' as an ARRAY"); + } + + private ParquetWriter createWriter(Path path, MessageType schema) throws IOException { + Configuration conf = new Configuration(); + return ExampleParquetWriter.builder( + HadoopOutputFile.fromPath( + new org.apache.hadoop.fs.Path(path.toString()), conf)) + .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) + .withConf(conf) + .withType(schema) + .build(); + } + + private RecordReader createReader(Path path, RowType readType) throws IOException { + ParquetReaderFactory factory = + new ParquetReaderFactory(new Options(), readType, 1024, null); + LocalFileIO fileIO = new LocalFileIO(); + return factory.createReader( + new FormatReaderContext(fileIO, path, fileIO.getFileSize(path), null, null)); + } +} diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetListLayoutResolverTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetListLayoutResolverTest.java new file mode 100644 index 000000000000..e190cda8016f --- /dev/null +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetListLayoutResolverTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.format.parquet; + +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; + +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for {@link ParquetListLayoutResolver}. + * + *

All list layout decisions should be made through {@link ParquetListLayoutResolver} so that + * schema inference, requested-schema clipping and reader construction share a single + * interpretation; the layouts below follow the backward-compatibility rules in the Parquet spec. + */ +public class ParquetListLayoutResolverTest { + + @Test + public void testResolveThreeLevelElementType() { + // Rule 5: canonical three-level list (list -> element) with a primitive element. + GroupType threeLevelPrimitiveList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("element") + .named("list")) + .named("my_list"); + Type threeLevelPrimitiveElement = + ParquetListLayoutResolver.resolveElementType(threeLevelPrimitiveList); + assertThat(threeLevelPrimitiveElement.isPrimitive()).isEqualTo(true); + assertThat(threeLevelPrimitiveElement.getName()).isEqualTo("element"); + + // Rule 5: canonical three-level list (list -> element) with a group element. + GroupType threeLevelGroupElement = + Types.buildGroup(Type.Repetition.OPTIONAL) + .optional(INT32) + .named("x") + .optional(INT32) + .named("y") + .named("element"); + GroupType threeLevelGroupList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .addField(threeLevelGroupElement) + .named("list")) + .named("my_list"); + Type threeLevelGroupElementType = + ParquetListLayoutResolver.resolveElementType(threeLevelGroupList); + assertThat(threeLevelGroupElementType.isPrimitive()).isEqualTo(false); + assertThat(threeLevelGroupElementType.getName()).isEqualTo("element"); + assertThat(threeLevelGroupElementType.asGroupType().getFieldCount()).isEqualTo(2); + } + + @Test + public void testResolveTwoLevelElementType() { + // Rule 1: a repeated primitive field is itself the element type. + GroupType twoLevelPrimitiveList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .repeated(INT32) + .named("element") + .named("my_list"); + Type twoLevelPrimitiveElement = + ParquetListLayoutResolver.resolveElementType(twoLevelPrimitiveList); + assertThat(twoLevelPrimitiveElement.isPrimitive()).isEqualTo(true); + assertThat(twoLevelPrimitiveElement.getName()).isEqualTo("element"); + + // Rule 2: a repeated group with multiple fields is itself the element type. + GroupType twoLevelGroupElement = + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("x") + .optional(INT32) + .named("y") + .named("element"); + GroupType twoLevelGroupList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(twoLevelGroupElement) + .named("my_list"); + Type twoLevelGroupElementType = + ParquetListLayoutResolver.resolveElementType(twoLevelGroupList); + assertThat(twoLevelGroupElementType.isPrimitive()).isEqualTo(false); + assertThat(twoLevelGroupElementType.getName()).isEqualTo("element"); + assertThat(twoLevelGroupElementType.asGroupType().getFieldCount()).isEqualTo(2); + } + + @Test + public void testResolveLegacyNestedElementType() { + // Rule 3: a repeated group with a single repeated field is the element type. + GroupType nestedRepeatedWrapper = + Types.buildGroup(Type.Repetition.REPEATED) + .repeated(INT32) + .named("array") + .named("array"); + GroupType nestedRepeatedList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(nestedRepeatedWrapper) + .named("my_list"); + Type nestedRepeatedElement = + ParquetListLayoutResolver.resolveElementType(nestedRepeatedList); + assertThat(nestedRepeatedElement.isPrimitive()).isEqualTo(false); + assertThat(nestedRepeatedElement.getName()).isEqualTo("array"); + assertThat(nestedRepeatedElement.asGroupType().getFieldCount()).isEqualTo(1); + assertThat(nestedRepeatedElement.asGroupType().getType(0).getName()).isEqualTo("array"); + } + + @Test + public void testResolveLegacyNamedElementType() { + // Rule 4: a repeated group named "array" with one field is the element type. + GroupType arrayWrapper = + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("foo") + .named("array"); + GroupType arrayWrapperList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(arrayWrapper) + .named("my_list"); + Type arrayWrapperElement = ParquetListLayoutResolver.resolveElementType(arrayWrapperList); + assertThat(arrayWrapperElement.isPrimitive()).isEqualTo(false); + assertThat(arrayWrapperElement.getName()).isEqualTo("array"); + + // Rule 4: a repeated group named "_tuple" with one field is the element type. + GroupType tupleWrapper = + Types.buildGroup(Type.Repetition.REPEATED) + .required(BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("str") + .named("my_list_tuple"); + GroupType tupleWrapperList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(tupleWrapper) + .named("my_list"); + Type tupleWrapperElement = ParquetListLayoutResolver.resolveElementType(tupleWrapperList); + assertThat(tupleWrapperElement.isPrimitive()).isEqualTo(false); + assertThat(tupleWrapperElement.getName()).isEqualTo("my_list_tuple"); + assertThat(tupleWrapperElement.asGroupType().getFieldCount()).isEqualTo(1); + assertThat(tupleWrapperElement.asGroupType().getType(0).getName()).isEqualTo("str"); + } + + @Test + public void testResolveLegacyBagElementType() { + // Rule 5: a repeated group with a single non-repeated field that is neither "array" nor + // "_tuple" unwraps to the single child (e.g. Hive's bag/array_element encoding). + GroupType bagWrapper = + Types.buildGroup(Type.Repetition.REPEATED) + .optional(INT32) + .named("array_element") + .named("bag"); + GroupType bagWrapperList = + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField(bagWrapper) + .named("my_list"); + assertThat(ParquetListLayoutResolver.isThreeLevelList(bagWrapperList)).isTrue(); + assertThat(ParquetListLayoutResolver.isCanonicalList(bagWrapperList)).isFalse(); + Type bagWrapperElement = ParquetListLayoutResolver.resolveElementType(bagWrapperList); + assertThat(bagWrapperElement.isPrimitive()).isEqualTo(true); + assertThat(bagWrapperElement.getName()).isEqualTo("array_element"); + } +} diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java index f40550831dff..4a23e405b8af 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetSchemaConverterTest.java @@ -39,6 +39,7 @@ import static org.apache.paimon.format.parquet.ParquetSchemaConverter.convertToParquetMessageType; import static org.apache.paimon.types.DataTypesTest.assertThat; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT64; /** Test for {@link ParquetSchemaConverter}. */ @@ -206,4 +207,196 @@ public void testGeographyLogicalTypeDefaults() { assertThat(expected).isEqualTo(convertToPaimonRowType(messageType)); } + + /** + * Backward-compatibility Rules 1, 2 and 4: two-level lists (a repeated primitive, a repeated + * struct, and a legacy {@code array} wrapper) infer a non-nullable element, because a {@code + * REPEATED} node is never null. This matches parquet-cpp's {@code SchemaManifest} contract and + * keeps the inferred schema symmetric with what Paimon's own writer produces. + * + *

Rule 5 (three-level wrapper) is the contrast case: the wrapper child's own nullability is + * preserved, so an {@code OPTIONAL} element stays nullable. + */ + @Test + public void testInferTwoLevelListElementNotNull() { + // Rule 1: optional group my_list (LIST) { repeated int32 element; } + MessageType rule1 = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.primitive(INT32, Type.Repetition.REPEATED) + .named("element") + .withId(1)) + .named("my_list") + .withId(0)); + assertThat( + new RowType( + Arrays.asList( + new DataField( + 0, + "my_list", + new ArrayType(DataTypes.INT().notNull()))))) + .isEqualTo(convertToPaimonRowType(rule1)); + + // Rule 2: optional group my_list (LIST) { repeated group element { optional int32 x; + // optional int32 y; } } + MessageType rule2 = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .addField( + Types.primitive( + INT32, + Type.Repetition.OPTIONAL) + .named("x") + .withId(2)) + .addField( + Types.primitive( + INT32, + Type.Repetition.OPTIONAL) + .named("y") + .withId(3)) + .named("element") + .withId(1)) + .named("my_list") + .withId(0)); + RowType rule2Element = + new RowType( + Arrays.asList( + new DataField(2, "x", DataTypes.INT()), + new DataField(3, "y", DataTypes.INT()))); + assertThat( + new RowType( + Arrays.asList( + new DataField( + 0, + "my_list", + new ArrayType(rule2Element.notNull()))))) + .isEqualTo(convertToPaimonRowType(rule2)); + + // Rule 4: optional group my_list (LIST) { repeated group array { optional int32 foo; } } + MessageType rule4 = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .addField( + Types.primitive( + INT32, + Type.Repetition.OPTIONAL) + .named("foo") + .withId(2)) + .named("array") + .withId(1)) + .named("my_list") + .withId(0)); + RowType rule4Element = new RowType(Arrays.asList(new DataField(2, "foo", DataTypes.INT()))); + assertThat( + new RowType( + Arrays.asList( + new DataField( + 0, + "my_list", + new ArrayType(rule4Element.notNull()))))) + .isEqualTo(convertToPaimonRowType(rule4)); + + // Rule 5 (contrast): optional group my_list (LIST) { repeated group bag { optional int32 + // array_element; } } keeps the OPTIONAL element nullable. + MessageType rule5 = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .addField( + Types.primitive( + INT32, + Type.Repetition.OPTIONAL) + .named("array_element") + .withId(2)) + .named("bag") + .withId(1)) + .named("my_list") + .withId(0)); + assertThat( + new RowType( + Arrays.asList( + new DataField( + 0, "my_list", new ArrayType(DataTypes.INT()))))) + .isEqualTo(convertToPaimonRowType(rule5)); + } + + /** + * Backward-compatibility Rule 3: an annotated list whose element is a nested legacy list infers + * {@code ARRAY NOT NULL>}. + */ + @Test + public void testInferNestedLegacyList() { + // Rule 3: optional group my_list (LIST) { repeated group element { repeated int32 array; } + // } + MessageType annotated = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.OPTIONAL) + .as(LogicalTypeAnnotation.listType()) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .addField( + Types.primitive( + INT32, + Type.Repetition.REPEATED) + .named("array") + .withId(2)) + .named("element") + .withId(1)) + .named("my_list") + .withId(0)); + assertThat( + new RowType( + Arrays.asList( + new DataField( + 0, + "my_list", + new ArrayType( + new ArrayType(DataTypes.INT().notNull()) + .notNull()))))) + .isEqualTo(convertToPaimonRowType(annotated)); + + // Without the annotation: repeated group my_list { repeated group array { optional int32 + // x; } } infers ARRAY NOT NULL>. + MessageType unannotated = + new MessageType( + "origin-parquet", + Types.buildGroup(Type.Repetition.REPEATED) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .addField( + Types.primitive( + INT32, + Type.Repetition.OPTIONAL) + .named("x") + .withId(2)) + .named("array") + .withId(1)) + .named("my_list") + .withId(0)); + RowType unannotatedElement = + new RowType(Arrays.asList(new DataField(2, "x", DataTypes.INT()))); + assertThat( + new RowType( + Arrays.asList( + new DataField( + 0, + "my_list", + new ArrayType(unannotatedElement.notNull()))))) + .isEqualTo(convertToPaimonRowType(unannotated)); + } }