From e4688b96fbfb7e5a6a5fa9183f7727a751529e2f Mon Sep 17 00:00:00 2001 From: SreeramaYeshwanthGowd Date: Fri, 24 Jul 2026 12:15:53 +0530 Subject: [PATCH 1/4] [SPARK-51975][SQL][PYTHON] Add variant_from_arrays and variant_from_entries --- .../reference/pyspark.sql/functions.rst | 2 + .../pyspark/sql/connect/functions/builtin.py | 14 ++ python/pyspark/sql/functions/__init__.py | 2 + python/pyspark/sql/functions/builtin.py | 69 ++++++ python/pyspark/sql/tests/test_functions.py | 17 ++ .../org/apache/spark/sql/functions.scala | 30 +++ .../catalyst/analysis/FunctionRegistry.scala | 2 + .../variant/VariantExpressionEvalUtils.scala | 55 +++++ .../variant/variantExpressions.scala | 147 +++++++++++- .../variant/VariantExpressionSuite.scala | 57 +++++ .../sql-functions/sql-expression-schema.md | 2 + .../variant-from-arrays-entries.sql.out | 182 +++++++++++++++ .../variant/variant-from-arrays-entries.sql | 39 ++++ .../variant-from-arrays-entries.sql.out | 221 ++++++++++++++++++ .../spark/sql/VariantEndToEndSuite.scala | 25 ++ 15 files changed, 863 insertions(+), 1 deletion(-) create mode 100644 sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out diff --git a/python/docs/source/reference/pyspark.sql/functions.rst b/python/docs/source/reference/pyspark.sql/functions.rst index 7ce413fde0bb9..9dc306b8f22f0 100644 --- a/python/docs/source/reference/pyspark.sql/functions.rst +++ b/python/docs/source/reference/pyspark.sql/functions.rst @@ -605,6 +605,8 @@ VARIANT Functions variant_array_append try_variant_array_append variant_delete + variant_from_arrays + variant_from_entries variant_get variant_insert try_variant_insert diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py index acf1ee6b069fa..69266136487af 100644 --- a/python/pyspark/sql/connect/functions/builtin.py +++ b/python/pyspark/sql/connect/functions/builtin.py @@ -2185,6 +2185,20 @@ def to_variant_object(col: "ColumnOrName") -> Column: to_variant_object.__doc__ = pysparkfuncs.to_variant_object.__doc__ +def variant_from_arrays(keys: "ColumnOrName", values: "ColumnOrName") -> Column: + return _invoke_function_over_columns("variant_from_arrays", keys, values) + + +variant_from_arrays.__doc__ = pysparkfuncs.variant_from_arrays.__doc__ + + +def variant_from_entries(entries: "ColumnOrName") -> Column: + return _invoke_function_over_columns("variant_from_entries", entries) + + +variant_from_entries.__doc__ = pysparkfuncs.variant_from_entries.__doc__ + + def parse_json(col: "ColumnOrName") -> Column: return _invoke_function("parse_json", _to_col(col)) diff --git a/python/pyspark/sql/functions/__init__.py b/python/pyspark/sql/functions/__init__.py index 7780bae38ee94..247b2ae3cd95c 100644 --- a/python/pyspark/sql/functions/__init__.py +++ b/python/pyspark/sql/functions/__init__.py @@ -483,6 +483,8 @@ "variant_array_append", "try_variant_array_append", "variant_delete", + "variant_from_arrays", + "variant_from_entries", "variant_get", "variant_insert", "try_variant_insert", diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 3cd42dc2e0e28..8bf162d6dd098 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -22499,6 +22499,75 @@ def to_variant_object( return _invoke_function("to_variant_object", _to_java_column(col)) +@_try_remote_functions +def variant_from_arrays(keys: "ColumnOrName", values: "ColumnOrName") -> Column: + """ + Creates a variant object from the given arrays of keys and values. The keys must be non-null + strings and the two arrays must have the same length. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + keys : :class:`~pyspark.sql.Column` or column name + an array of string keys. + values : :class:`~pyspark.sql.Column` or column name + an array of values. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + + See Also + -------- + :meth:`pyspark.sql.functions.variant_from_entries` + :meth:`pyspark.sql.functions.to_variant_object` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT array('a', 'b') AS keys, array(1, 2) AS values") + >>> df.select(sf.variant_from_arrays("keys", "values").cast("string").alias("r")).collect() + [Row(r='{"a":1,"b":2}')] + """ + return _invoke_function_over_columns("variant_from_arrays", keys, values) + + +@_try_remote_functions +def variant_from_entries(entries: "ColumnOrName") -> Column: + """ + Creates a variant object from an array of key/value struct entries. The keys must be non-null + strings. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + entries : :class:`~pyspark.sql.Column` or column name + an array of key/value structs, where the first field is a string key and the second field + is the value. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + + See Also + -------- + :meth:`pyspark.sql.functions.variant_from_arrays` + :meth:`pyspark.sql.functions.to_variant_object` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT array(struct('a', 1), struct('b', 2)) AS entries") + >>> df.select(sf.variant_from_entries("entries").cast("string").alias("r")).collect() + [Row(r='{"a":1,"b":2}')] + """ + return _invoke_function_over_columns("variant_from_entries", entries) + + @_try_remote_functions def parse_json( col: "ColumnOrName", diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py index 2594c8c60b760..6aab6c9deb089 100644 --- a/python/pyspark/sql/tests/test_functions.py +++ b/python/pyspark/sql/tests/test_functions.py @@ -3661,6 +3661,23 @@ def test_to_variant_object(self): ).collect() self.assertEqual("""{"a":1}""", actual[0]["var"]) + def test_variant_from_arrays_and_entries(self): + df = self.spark.createDataFrame( + [(["a", "b"], [1, 2])], "keys array, values array" + ) + actual = df.select( + F.to_json(F.variant_from_arrays("keys", "values")).alias("var"), + ).collect() + self.assertEqual("""{"a":1,"b":2}""", actual[0]["var"]) + + df2 = self.spark.createDataFrame( + [([("a", 1), ("b", 2)],)], "entries array>" + ) + actual2 = df2.select( + F.to_json(F.variant_from_entries("entries")).alias("var"), + ).collect() + self.assertEqual("""{"a":1,"b":2}""", actual2[0]["var"]) + def test_schema_of_csv(self): with self.assertRaises(PySparkTypeError) as pe: F.schema_of_csv(1) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 8b8cd98960e0b..2225900491859 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -14081,6 +14081,36 @@ object functions { */ def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) + /** + * Creates a variant object from the given arrays of keys and values. The keys must be non-null + * strings and the two arrays must have the same length. + * + * @param keys + * a column that evaluates to an array of string keys. + * @param values + * a column that evaluates to an array of values. + * @group variant_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. + */ + def variant_from_arrays(keys: Column, values: Column): Column = + Column.fn("variant_from_arrays", keys, values) + + /** + * Creates a variant object from an array of key/value struct entries. The keys must be non-null + * strings. + * + * @param entries + * a column that evaluates to an array of key/value structs. + * @group variant_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. + */ + def variant_from_entries(entries: Column): Column = + Column.fn("variant_from_entries", entries) + /** * Check if a variant value is a variant null. Returns true if and only if the input is a * variant null and false otherwise (including in the case of SQL NULL). diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 5b39aa06f1e41..748f9ff24cc08 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -985,6 +985,8 @@ object FunctionRegistry { expression[SchemaOfVariant]("schema_of_variant"), expression[SchemaOfVariantAgg]("schema_of_variant_agg"), expression[ToVariantObject]("to_variant_object"), + expression[VariantFromArrays]("variant_from_arrays"), + expression[VariantFromEntries]("variant_from_entries"), expression[IsValidVariant]("is_valid_variant"), expression[VariantDelete]("variant_delete"), expressionBuilder("variant_insert", VariantInsertExpressionBuilder), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala index 0872cda998013..357596beac4b7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionEvalUtils.scala @@ -274,6 +274,61 @@ object VariantExpressionEvalUtils { new VariantVal(v.getValue, v.getMetadata) } + /** + * Build a variant object directly from a keys array and a values array, without materializing an + * intermediate map. Keys must be non-null strings and the two arrays must have equal length. A + * null key raises `NULL_MAP_KEY`, a duplicate key raises `VARIANT_DUPLICATE_KEY` (matching + * to_variant_object), and null values are kept as variant null. + */ + def variantFromArrays(keys: ArrayData, values: ArrayData, valueType: DataType): VariantVal = { + if (keys.numElements() != values.numElements()) { + // Reuse the same error map_from_arrays raises for a keys/values length mismatch. + throw QueryExecutionErrors.mapDataKeyArrayLengthDiffersFromValueArrayLengthError() + } + val builder = new VariantBuilder(false) + val start = builder.getWritePos + val fields = new java.util.ArrayList[VariantBuilder.FieldEntry](keys.numElements()) + for (i <- 0 until keys.numElements()) { + if (keys.isNullAt(i)) { + throw QueryExecutionErrors.nullAsMapKeyNotAllowedError() + } + val key = keys.getUTF8String(i).toString + val id = builder.addKey(key) + fields.add(new VariantBuilder.FieldEntry(key, id, builder.getWritePos - start)) + val value = if (values.isNullAt(i)) null else values.get(i, valueType) + buildVariant(builder, value, valueType) + } + builder.finishWritingObject(start, fields) + val v = builder.result() + new VariantVal(v.getValue, v.getMetadata) + } + + /** + * Build a variant object directly from an array of key/value struct entries, without an + * intermediate map. Keys must be non-null strings. A null key raises `NULL_MAP_KEY`, a + * duplicate key raises `VARIANT_DUPLICATE_KEY`, null values are kept as variant null, and null + * entries are handled by the caller. + */ + def variantFromEntries(entries: ArrayData, valueType: DataType): VariantVal = { + val builder = new VariantBuilder(false) + val start = builder.getWritePos + val fields = new java.util.ArrayList[VariantBuilder.FieldEntry](entries.numElements()) + for (i <- 0 until entries.numElements()) { + val entry = entries.getStruct(i, 2) + if (entry.isNullAt(0)) { + throw QueryExecutionErrors.nullAsMapKeyNotAllowedError() + } + val key = entry.getUTF8String(0).toString + val id = builder.addKey(key) + fields.add(new VariantBuilder.FieldEntry(key, id, builder.getWritePos - start)) + val value = if (entry.isNullAt(1)) null else entry.get(1, valueType) + buildVariant(builder, value, valueType) + } + builder.finishWritingObject(start, fields) + val v = builder.result() + new VariantVal(v.getValue, v.getMetadata) + } + /** Returns `true` if a data type is or has a child variant type. */ def typeContainsVariant(dt: DataType): Boolean = dt match { case _: VariantType => true diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala index fa6a955097d99..060445ef2e3d1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.json.JsonInferSchema import org.apache.spark.sql.catalyst.plans.logical.{FunctionSignature, InputParameter} import org.apache.spark.sql.catalyst.trees.TreePattern.{TreePattern, VARIANT_GET} import org.apache.spark.sql.catalyst.trees.UnaryLike -import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, QuotingUtils} +import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, GenericArrayData, QuotingUtils} import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryErrorsBase, QueryExecutionErrors} import org.apache.spark.sql.internal.SQLConf @@ -185,6 +185,151 @@ case class ToVariantObject(child: Expression) } } +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = "_FUNC_(keys, values) - Creates a variant object from the given arrays of keys and values. The keys must be non-null strings and the two arrays must have the same length.", + examples = """ + Examples: + > SELECT _FUNC_(array('a', 'b'), array(1, 2)); + {"a":1,"b":2} + """, + since = "4.3.0", + group = "variant_funcs") +// scalastyle:on line.size.limit +case class VariantFromArrays(left: Expression, right: Expression) + extends BinaryExpression + with ExpectsInputTypes + with QueryErrorsBase { + override def nullIntolerant: Boolean = true + override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType, ArrayType) + override def dataType: DataType = VariantType + + private lazy val valueType: DataType = right.dataType.asInstanceOf[ArrayType].elementType + + override def checkInputDataTypes(): TypeCheckResult = { + val defaultCheck = super.checkInputDataTypes() + if (defaultCheck.isFailure) { + defaultCheck + } else { + left.dataType.asInstanceOf[ArrayType].elementType match { + case _: StringType if VariantGet.checkDataType(valueType, allowStructsAndMaps = true) => + TypeCheckResult.TypeCheckSuccess + case _: StringType => + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = + Map("srcType" -> toSQLType(valueType), "targetType" -> toSQLType(VariantType))) + case _ => + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> ordinalNumber(0), + "requiredType" -> toSQLType(ArrayType(StringType)), + "inputSql" -> toSQLExpr(left), + "inputType" -> toSQLType(left.dataType))) + } + } + } + + override def prettyName: String = "variant_from_arrays" + + override protected def withNewChildrenInternal( + newLeft: Expression, newRight: Expression): VariantFromArrays = + copy(left = newLeft, right = newRight) + + override protected def nullSafeEval(keyArray: Any, valueArray: Any): Any = + VariantExpressionEvalUtils.variantFromArrays( + keyArray.asInstanceOf[ArrayData], valueArray.asInstanceOf[ArrayData], valueType) + + override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + nullSafeCodeGen(ctx, ev, (keyArray, valueArray) => { + val cls = variant.VariantExpressionEvalUtils.getClass.getName.stripSuffix("$") + val valueTypeArg = ctx.addReferenceObj("valueType", valueType) + s"${ev.value} = $cls.variantFromArrays($keyArray, $valueArray, $valueTypeArg);" + }) + } +} + +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = "_FUNC_(entries) - Creates a variant object from an array of key/value struct entries. The keys must be non-null strings.", + examples = """ + Examples: + > SELECT _FUNC_(array(struct('a', 1), struct('b', 2))); + {"a":1,"b":2} + """, + since = "4.3.0", + group = "variant_funcs") +// scalastyle:on line.size.limit +case class VariantFromEntries(child: Expression) + extends UnaryExpression + with QueryErrorsBase { + override def nullIntolerant: Boolean = true + + @transient + private lazy val dataTypeDetails: Option[(DataType, Boolean)] = child.dataType match { + case ArrayType( + StructType(Array(StructField(_, _, _, _), StructField(_, valueType, _, _))), + containsNull) => + Some((valueType, containsNull)) + case _ => None + } + + @transient private lazy val valueType: DataType = dataTypeDetails.get._1 + @transient private lazy val nullEntries: Boolean = dataTypeDetails.get._2 + + override def nullable: Boolean = child.nullable || nullEntries + override def dataType: DataType = VariantType + + override def checkInputDataTypes(): TypeCheckResult = child.dataType match { + case ArrayType( + StructType(Array(StructField(_, _: StringType, _, _), StructField(_, vt, _, _))), _) => + if (VariantGet.checkDataType(vt, allowStructsAndMaps = true)) { + TypeCheckResult.TypeCheckSuccess + } else { + DataTypeMismatch( + errorSubClass = "CAST_WITHOUT_SUGGESTION", + messageParameters = + Map("srcType" -> toSQLType(vt), "targetType" -> toSQLType(VariantType))) + } + case _ => + DataTypeMismatch( + errorSubClass = "UNEXPECTED_INPUT_TYPE", + messageParameters = Map( + "paramIndex" -> ordinalNumber(0), + "requiredType" -> s"${toSQLType(ArrayType)} of pair ${toSQLType(StructType)}", + "inputSql" -> toSQLExpr(child), + "inputType" -> toSQLType(child.dataType))) + } + + override def prettyName: String = "variant_from_entries" + + override protected def withNewChildInternal(newChild: Expression): VariantFromEntries = + copy(child = newChild) + + override protected def nullSafeEval(input: Any): Any = { + val entries = input.asInstanceOf[ArrayData] + if (nullEntries) { + var i = 0 + while (i < entries.numElements()) { + if (entries.isNullAt(i)) return null + i += 1 + } + } + VariantExpressionEvalUtils.variantFromEntries(entries, valueType) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + nullSafeCodeGen(ctx, ev, c => { + val cls = variant.VariantExpressionEvalUtils.getClass.getName.stripSuffix("$") + val valueTypeArg = ctx.addReferenceObj("valueType", valueType) + ctx.nullArrayElementsSaveExec(nullEntries, ev.isNull, c) { + s"${ev.value} = $cls.variantFromEntries($c, $valueTypeArg);" + } + }) + } +} + // A path segment in the `VariantGet` expression represents either an object key access or an array // index access. sealed abstract class VariantPathSegment extends Serializable diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala index 92f20bcc31272..41b007a76fc4a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala @@ -976,6 +976,63 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { checkFailure(Map(1 -> 1), toVariantObject = true) } + test("variant_from_arrays and variant_from_entries") { + def keysValues(keys: Any, values: Any, valueType: DataType): VariantFromArrays = + VariantFromArrays( + Literal.create(keys, ArrayType(StringType)), + Literal.create(values, ArrayType(valueType))) + + def entriesOf(entries: Any, valueType: DataType, + containsNull: Boolean = false): VariantFromEntries = + VariantFromEntries(Literal.create(entries, ArrayType( + StructType(Seq(StructField("k", StringType), StructField("v", valueType))), containsNull))) + + // Basic object construction; keys are sorted in the resulting variant object. + checkEvaluation(StructsToJson(Map.empty, + keysValues(Array("z", "a"), Array(1, 2), IntegerType)), """{"a":2,"z":1}""") + checkEvaluation(StructsToJson(Map.empty, + entriesOf(Array(Row("a", 1), Row("b", 2)), IntegerType)), """{"a":1,"b":2}""") + + // Empty input produces an empty object. + checkEvaluation(StructsToJson(Map.empty, + keysValues(Array.empty[String], Array.empty[Int], IntegerType)), "{}") + + // Null values are kept as variant null; nested values are converted recursively. + checkEvaluation(StructsToJson(Map.empty, + entriesOf(Array(Row("a", 1), Row("b", null)), IntegerType)), """{"a":1,"b":null}""") + checkEvaluation(StructsToJson(Map.empty, + keysValues(Array("a"), Array(Array(1, 2, 3)), ArrayType(IntegerType))), """{"a":[1,2,3]}""") + checkEvaluation(StructsToJson(Map.empty, keysValues(Array("a"), Array(Row(1)), + StructType(Seq(StructField("i", IntegerType))))), """{"a":{"i":1}}""") + + // A null entry makes the whole result null. + checkEvaluation(StructsToJson(Map.empty, + entriesOf(Array(Row("a", 1), null), IntegerType, containsNull = true)), null) + + // A null array input produces null. + checkEvaluation(StructsToJson(Map.empty, VariantFromArrays( + Literal.create(null, ArrayType(StringType)), + Literal.create(Array(1), ArrayType(IntegerType)))), null) + + // A null key is rejected. + checkErrorInExpression[SparkRuntimeException]( + keysValues(Array("a", null), Array(1, 2), IntegerType), + "NULL_MAP_KEY", Map.empty[String, String]) + + // Duplicate keys are rejected for both forms. + checkErrorInExpression[SparkRuntimeException]( + keysValues(Array("a", "a"), Array(1, 2), IntegerType), + "VARIANT_DUPLICATE_KEY", Map("key" -> "a")) + checkErrorInExpression[SparkRuntimeException]( + entriesOf(Array(Row("a", 1), Row("a", 2)), IntegerType), + "VARIANT_DUPLICATE_KEY", Map("key" -> "a")) + + // Mismatched array lengths are rejected. + checkErrorInExpression[SparkRuntimeException]( + keysValues(Array("a", "b"), Array(1), IntegerType), + "_LEGACY_ERROR_TEMP_2128", Map.empty[String, String]) + } + test("schema_of_variant - unknown type") { val emptyMetadata = Array[Byte](VERSION, 0, 0) diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 99285d126acdf..d0cb91d2a2087 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -560,6 +560,8 @@ | org.apache.spark.sql.catalyst.expressions.variant.TryVariantInsertExpressionBuilder | try_variant_insert | SELECT try_variant_insert(parse_json('{"a": 1}'), '$.b', 2) | struct | | org.apache.spark.sql.catalyst.expressions.variant.VariantArrayAppendExpressionBuilder | variant_array_append | SELECT variant_array_append(parse_json('[1, 2, 3]'), '$', 4) | struct | | org.apache.spark.sql.catalyst.expressions.variant.VariantDelete | variant_delete | SELECT variant_delete(parse_json('{"a": 1, "b": 2, "c": 3, "items": [1, 2, 3]}'), NULL, '$.a', '$.c') | struct | +| org.apache.spark.sql.catalyst.expressions.variant.VariantFromArrays | variant_from_arrays | SELECT variant_from_arrays(array('a', 'b'), array(1, 2)) | struct | +| org.apache.spark.sql.catalyst.expressions.variant.VariantFromEntries | variant_from_entries | SELECT variant_from_entries(array(struct('a', 1), struct('b', 2))) | struct | | org.apache.spark.sql.catalyst.expressions.variant.VariantGetExpressionBuilder | variant_get | SELECT variant_get(parse_json('{"a": 1}'), '$.a', 'int') | struct | | org.apache.spark.sql.catalyst.expressions.variant.VariantInsertExpressionBuilder | variant_insert | SELECT variant_insert(parse_json('{"a": 1}'), '$.b', 2) | struct | | org.apache.spark.sql.catalyst.expressions.variant.VariantSetExpressionBuilder | variant_set | SELECT variant_set(parse_json('{"a": 1}'), '$.a', 2) | struct | diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out new file mode 100644 index 0000000000000..96288236a84dc --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out @@ -0,0 +1,182 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +select cast(variant_from_arrays(array('z', 'a'), array(1, 2)) as string) +-- !query analysis +Project [cast(variant_from_arrays(array(z, a), array(1, 2)) as string) AS CAST(variant_from_arrays(array(z, a), array(1, 2)) AS STRING)#x] ++- OneRowRelation + + +-- !query +select cast(variant_from_arrays(cast(array() as array), cast(array() as array)) as string) +-- !query analysis +Project [cast(variant_from_arrays(cast(array() as array), cast(array() as array)) as string) AS CAST(variant_from_arrays(array(), array()) AS STRING)#x] ++- OneRowRelation + + +-- !query +select cast(variant_from_arrays(array('a', 'b'), array(1, cast(null as int))) as string) +-- !query analysis +Project [cast(variant_from_arrays(array(a, b), array(1, cast(null as int))) as string) AS CAST(variant_from_arrays(array(a, b), array(1, CAST(NULL AS INT))) AS STRING)#x] ++- OneRowRelation + + +-- !query +select cast(variant_from_arrays(array('a'), array(array(1, 2, 3))) as string) +-- !query analysis +Project [cast(variant_from_arrays(array(a), array(array(1, 2, 3))) as string) AS CAST(variant_from_arrays(array(a), array(array(1, 2, 3))) AS STRING)#x] ++- OneRowRelation + + +-- !query +select cast(variant_from_arrays(cast(null as array), array(1)) as string) +-- !query analysis +Project [cast(variant_from_arrays(cast(null as array), array(1)) as string) AS CAST(variant_from_arrays(NULL, array(1)) AS STRING)#x] ++- OneRowRelation + + +-- !query +select variant_from_arrays(array('a', cast(null as string)), array(1, 2)) +-- !query analysis +Project [variant_from_arrays(array(a, cast(null as string)), array(1, 2)) AS variant_from_arrays(array(a, CAST(NULL AS STRING)), array(1, 2))#x] ++- OneRowRelation + + +-- !query +select variant_from_arrays(array('a', 'a'), array(1, 2)) +-- !query analysis +Project [variant_from_arrays(array(a, a), array(1, 2)) AS variant_from_arrays(array(a, a), array(1, 2))#x] ++- OneRowRelation + + +-- !query +select variant_from_arrays(array('a', 'b'), array(1)) +-- !query analysis +Project [variant_from_arrays(array(a, b), array(1)) AS variant_from_arrays(array(a, b), array(1))#x] ++- OneRowRelation + + +-- !query +select variant_from_arrays(array(1, 2), array('a', 'b')) +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"array(1, 2)\"", + "inputType" : "\"ARRAY\"", + "paramIndex" : "first", + "requiredType" : "\"ARRAY\"", + "sqlExpr" : "\"variant_from_arrays(array(1, 2), array(a, b))\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 56, + "fragment" : "variant_from_arrays(array(1, 2), array('a', 'b'))" + } ] +} + + +-- !query +select variant_from_arrays(array('a'), array(map(1, 2))) +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION", + "sqlState" : "42K09", + "messageParameters" : { + "sqlExpr" : "\"variant_from_arrays(array(a), array(map(1, 2)))\"", + "srcType" : "\"MAP\"", + "targetType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 56, + "fragment" : "variant_from_arrays(array('a'), array(map(1, 2)))" + } ] +} + + +-- !query +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', 1), named_struct('k', 'b', 'v', 2))) as string) +-- !query analysis +Project [cast(variant_from_entries(array(named_struct(k, a, v, 1), named_struct(k, b, v, 2))) as string) AS CAST(variant_from_entries(array(named_struct(k, a, v, 1), named_struct(k, b, v, 2))) AS STRING)#x] ++- OneRowRelation + + +-- !query +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', cast(null as int)))) as string) +-- !query analysis +Project [cast(variant_from_entries(array(named_struct(k, a, v, cast(null as int)))) as string) AS CAST(variant_from_entries(array(named_struct(k, a, v, CAST(NULL AS INT)))) AS STRING)#x] ++- OneRowRelation + + +-- !query +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', 1), cast(null as struct))) as string) +-- !query analysis +Project [cast(variant_from_entries(array(named_struct(k, a, v, 1), cast(null as struct))) as string) AS CAST(variant_from_entries(array(named_struct(k, a, v, 1), NULL)) AS STRING)#x] ++- OneRowRelation + + +-- !query +select cast(variant_from_entries(cast(null as array>)) as string) +-- !query analysis +Project [cast(variant_from_entries(cast(null as array>)) as string) AS CAST(variant_from_entries(NULL) AS STRING)#x] ++- OneRowRelation + + +-- !query +select variant_from_entries(array(named_struct('k', cast(null as string), 'v', 1))) +-- !query analysis +Project [variant_from_entries(array(named_struct(k, cast(null as string), v, 1))) AS variant_from_entries(array(named_struct(k, CAST(NULL AS STRING), v, 1)))#x] ++- OneRowRelation + + +-- !query +select variant_from_entries(array(1, 2)) +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"array(1, 2)\"", + "inputType" : "\"ARRAY\"", + "paramIndex" : "first", + "requiredType" : "\"ARRAY\" of pair \"STRUCT\"", + "sqlExpr" : "\"variant_from_entries(array(1, 2))\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 40, + "fragment" : "variant_from_entries(array(1, 2))" + } ] +} + + +-- !query +select variant_from_entries(array(named_struct('k', 'a', 'v', map(1, 2)))) +-- !query analysis +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION", + "sqlState" : "42K09", + "messageParameters" : { + "sqlExpr" : "\"variant_from_entries(array(named_struct(k, a, v, map(1, 2))))\"", + "srcType" : "\"MAP\"", + "targetType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 74, + "fragment" : "variant_from_entries(array(named_struct('k', 'a', 'v', map(1, 2))))" + } ] +} diff --git a/sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql b/sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql new file mode 100644 index 0000000000000..8519fbd752c17 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql @@ -0,0 +1,39 @@ +-- variant_from_arrays + +-- Basic object construction; keys are sorted in the resulting variant object. +select cast(variant_from_arrays(array('z', 'a'), array(1, 2)) as string); +-- Empty input produces an empty object. +select cast(variant_from_arrays(cast(array() as array), cast(array() as array)) as string); +-- Null values are kept as variant null. +select cast(variant_from_arrays(array('a', 'b'), array(1, cast(null as int))) as string); +-- Nested values are converted recursively. +select cast(variant_from_arrays(array('a'), array(array(1, 2, 3))) as string); +-- A null array input produces null. +select cast(variant_from_arrays(cast(null as array), array(1)) as string); +-- A null key is rejected. +select variant_from_arrays(array('a', cast(null as string)), array(1, 2)); +-- Duplicate keys are rejected. +select variant_from_arrays(array('a', 'a'), array(1, 2)); +-- Mismatched array lengths are rejected. +select variant_from_arrays(array('a', 'b'), array(1)); +-- A non-string key type is rejected. +select variant_from_arrays(array(1, 2), array('a', 'b')); +-- A value type that cannot be cast to variant is rejected. +select variant_from_arrays(array('a'), array(map(1, 2))); + +-- variant_from_entries + +-- Basic object construction from key/value struct entries. +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', 1), named_struct('k', 'b', 'v', 2))) as string); +-- Null values are kept as variant null. +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', cast(null as int)))) as string); +-- A null entry makes the whole result null. +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', 1), cast(null as struct))) as string); +-- A null array input produces null. +select cast(variant_from_entries(cast(null as array>)) as string); +-- A null key is rejected. +select variant_from_entries(array(named_struct('k', cast(null as string), 'v', 1))); +-- A non-array-of-pair-struct input is rejected. +select variant_from_entries(array(1, 2)); +-- A value type that cannot be cast to variant is rejected. +select variant_from_entries(array(named_struct('k', 'a', 'v', map(1, 2)))); diff --git a/sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out b/sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out new file mode 100644 index 0000000000000..9d68139db823b --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out @@ -0,0 +1,221 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +select cast(variant_from_arrays(array('z', 'a'), array(1, 2)) as string) +-- !query schema +struct +-- !query output +{"a":2,"z":1} + + +-- !query +select cast(variant_from_arrays(cast(array() as array), cast(array() as array)) as string) +-- !query schema +struct +-- !query output +{} + + +-- !query +select cast(variant_from_arrays(array('a', 'b'), array(1, cast(null as int))) as string) +-- !query schema +struct +-- !query output +{"a":1,"b":null} + + +-- !query +select cast(variant_from_arrays(array('a'), array(array(1, 2, 3))) as string) +-- !query schema +struct +-- !query output +{"a":[1,2,3]} + + +-- !query +select cast(variant_from_arrays(cast(null as array), array(1)) as string) +-- !query schema +struct +-- !query output +NULL + + +-- !query +select variant_from_arrays(array('a', cast(null as string)), array(1, 2)) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "NULL_MAP_KEY", + "sqlState" : "2200E" +} + + +-- !query +select variant_from_arrays(array('a', 'a'), array(1, 2)) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "VARIANT_DUPLICATE_KEY", + "sqlState" : "22023", + "messageParameters" : { + "key" : "a" + } +} + + +-- !query +select variant_from_arrays(array('a', 'b'), array(1)) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "_LEGACY_ERROR_TEMP_2128" +} + + +-- !query +select variant_from_arrays(array(1, 2), array('a', 'b')) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"array(1, 2)\"", + "inputType" : "\"ARRAY\"", + "paramIndex" : "first", + "requiredType" : "\"ARRAY\"", + "sqlExpr" : "\"variant_from_arrays(array(1, 2), array(a, b))\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 56, + "fragment" : "variant_from_arrays(array(1, 2), array('a', 'b'))" + } ] +} + + +-- !query +select variant_from_arrays(array('a'), array(map(1, 2))) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION", + "sqlState" : "42K09", + "messageParameters" : { + "sqlExpr" : "\"variant_from_arrays(array(a), array(map(1, 2)))\"", + "srcType" : "\"MAP\"", + "targetType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 56, + "fragment" : "variant_from_arrays(array('a'), array(map(1, 2)))" + } ] +} + + +-- !query +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', 1), named_struct('k', 'b', 'v', 2))) as string) +-- !query schema +struct +-- !query output +{"a":1,"b":2} + + +-- !query +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', cast(null as int)))) as string) +-- !query schema +struct +-- !query output +{"a":null} + + +-- !query +select cast(variant_from_entries(array(named_struct('k', 'a', 'v', 1), cast(null as struct))) as string) +-- !query schema +struct +-- !query output +NULL + + +-- !query +select cast(variant_from_entries(cast(null as array>)) as string) +-- !query schema +struct +-- !query output +NULL + + +-- !query +select variant_from_entries(array(named_struct('k', cast(null as string), 'v', 1))) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "NULL_MAP_KEY", + "sqlState" : "2200E" +} + + +-- !query +select variant_from_entries(array(1, 2)) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", + "sqlState" : "42K09", + "messageParameters" : { + "inputSql" : "\"array(1, 2)\"", + "inputType" : "\"ARRAY\"", + "paramIndex" : "first", + "requiredType" : "\"ARRAY\" of pair \"STRUCT\"", + "sqlExpr" : "\"variant_from_entries(array(1, 2))\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 40, + "fragment" : "variant_from_entries(array(1, 2))" + } ] +} + + +-- !query +select variant_from_entries(array(named_struct('k', 'a', 'v', map(1, 2)))) +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.catalyst.ExtendedAnalysisException +{ + "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION", + "sqlState" : "42K09", + "messageParameters" : { + "sqlExpr" : "\"variant_from_entries(array(named_struct(k, a, v, map(1, 2))))\"", + "srcType" : "\"MAP\"", + "targetType" : "\"VARIANT\"" + }, + "queryContext" : [ { + "objectType" : "", + "objectName" : "", + "startIndex" : 8, + "stopIndex" : 74, + "fragment" : "variant_from_entries(array(named_struct('k', 'a', 'v', map(1, 2))))" + } ] +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala index 2d26356890d28..30675e759428d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/VariantEndToEndSuite.scala @@ -246,6 +246,31 @@ class VariantEndToEndSuite extends SharedSparkSession { } } + test("variant_from_arrays and variant_from_entries - Codegen Support") { + Seq("CODEGEN_ONLY", "NO_CODEGEN").foreach { codegenMode => + withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { + val entryType = StructType(Array( + StructField("k", StringType), StructField("v", IntegerType))) + val schema = StructType(Array( + StructField("keys", ArrayType(StringType)), + StructField("values", ArrayType(IntegerType)), + StructField("entries", ArrayType(entryType)))) + // Source non-foldable rows so the operators' doGenCode is actually exercised under codegen. + val data = Seq(Row(Seq("a", "b"), Seq(1, 2), Seq(Row("a", 1), Row("b", 2)))) + val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema) + val arraysDF = df.select(variant_from_arrays(col("keys"), col("values")).cast("string")) + val entriesDF = df.select(variant_from_entries(col("entries")).cast("string")) + val wholeStage = codegenMode == "CODEGEN_ONLY" + assert(arraysDF.queryExecution.executedPlan.exists( + _.isInstanceOf[WholeStageCodegenExec]) == wholeStage) + assert(entriesDF.queryExecution.executedPlan.exists( + _.isInstanceOf[WholeStageCodegenExec]) == wholeStage) + checkAnswer(arraysDF, Row("""{"a":1,"b":2}""")) + checkAnswer(entriesDF, Row("""{"a":1,"b":2}""")) + } + } + } + test("schema_of_variant") { def check(json: String, expected: String): Unit = { val df = Seq(json).toDF("j").selectExpr("schema_of_variant(parse_json(j))") From 4704329368358b45ce95b8594dac4f35f299418d Mon Sep 17 00:00:00 2001 From: SreeramaYeshwanthGowd Date: Fri, 24 Jul 2026 15:12:43 +0530 Subject: [PATCH 2/4] [SPARK-51975][SQL] Skip variant_from_arrays/entries in CollationExpressionWalkerSuite --- .../sql/collation/CollationExpressionWalkerSuite.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationExpressionWalkerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationExpressionWalkerSuite.scala index dd22f647ab31d..cc5bbd5b3d75c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationExpressionWalkerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationExpressionWalkerSuite.scala @@ -391,7 +391,11 @@ class CollationExpressionWalkerSuite extends SharedSparkSession { "approx_top_k", "approx_top_k_accumulate", "approx_top_k_combine", - "approx_top_k_estimate" + "approx_top_k_estimate", + // The variant object embeds the key string, which preserves the input case for collated + // strings, so the result is not comparable across collations. + "variant_from_arrays", + "variant_from_entries" ) logInfo("Total number of expression: " + expressionCounter) From 6f6fcc2e6de3314f9d52bc39af6ed9834c6b97f8 Mon Sep 17 00:00:00 2001 From: SreeramaYeshwanthGowd Date: Sat, 25 Jul 2026 06:43:45 +0530 Subject: [PATCH 3/4] [SPARK-51975][SQL] Add duplicate-key test for variant_from_entries --- .../variant/variant-from-arrays-entries.sql.out | 7 +++++++ .../variant/variant-from-arrays-entries.sql | 2 ++ .../variant/variant-from-arrays-entries.sql.out | 15 +++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out index 96288236a84dc..bdf1a34c53bb0 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/variant/variant-from-arrays-entries.sql.out @@ -136,6 +136,13 @@ Project [variant_from_entries(array(named_struct(k, cast(null as string), v, 1)) +- OneRowRelation +-- !query +select variant_from_entries(array(named_struct('k', 'a', 'v', 1), named_struct('k', 'a', 'v', 2))) +-- !query analysis +Project [variant_from_entries(array(named_struct(k, a, v, 1), named_struct(k, a, v, 2))) AS variant_from_entries(array(named_struct(k, a, v, 1), named_struct(k, a, v, 2)))#x] ++- OneRowRelation + + -- !query select variant_from_entries(array(1, 2)) -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql b/sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql index 8519fbd752c17..04f5565112999 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/variant/variant-from-arrays-entries.sql @@ -33,6 +33,8 @@ select cast(variant_from_entries(array(named_struct('k', 'a', 'v', 1), cast(null select cast(variant_from_entries(cast(null as array>)) as string); -- A null key is rejected. select variant_from_entries(array(named_struct('k', cast(null as string), 'v', 1))); +-- Duplicate keys are rejected. +select variant_from_entries(array(named_struct('k', 'a', 'v', 1), named_struct('k', 'a', 'v', 2))); -- A non-array-of-pair-struct input is rejected. select variant_from_entries(array(1, 2)); -- A value type that cannot be cast to variant is rejected. diff --git a/sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out b/sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out index 9d68139db823b..6a6442b6ccb33 100644 --- a/sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/variant/variant-from-arrays-entries.sql.out @@ -171,6 +171,21 @@ org.apache.spark.SparkRuntimeException } +-- !query +select variant_from_entries(array(named_struct('k', 'a', 'v', 1), named_struct('k', 'a', 'v', 2))) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkRuntimeException +{ + "errorClass" : "VARIANT_DUPLICATE_KEY", + "sqlState" : "22023", + "messageParameters" : { + "key" : "a" + } +} + + -- !query select variant_from_entries(array(1, 2)) -- !query schema From e388b4e4cc8e94653145985bdfc70b491629dc1f Mon Sep 17 00:00:00 2001 From: SreeramaYeshwanthGowd Date: Mon, 27 Jul 2026 05:54:22 +0530 Subject: [PATCH 4/4] [SPARK-51975][SQL] Add arguments docs to variant_from_arrays and variant_from_entries --- .../expressions/variant/variantExpressions.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala index 060445ef2e3d1..b4133dfebc713 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala @@ -188,6 +188,11 @@ case class ToVariantObject(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(keys, values) - Creates a variant object from the given arrays of keys and values. The keys must be non-null strings and the two arrays must have the same length.", + arguments = """ + Arguments: + * keys - An array of non-null strings used as the object keys. + * values - An array of values, with the same length as the keys array. + """, examples = """ Examples: > SELECT _FUNC_(array('a', 'b'), array(1, 2)); @@ -253,6 +258,10 @@ case class VariantFromArrays(left: Expression, right: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(entries) - Creates a variant object from an array of key/value struct entries. The keys must be non-null strings.", + arguments = """ + Arguments: + * entries - An array of key/value structs, where the first field is a non-null string key. + """, examples = """ Examples: > SELECT _FUNC_(array(struct('a', 1), struct('b', 2)));