diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 82f81861a4dc..11d3738c1a3e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -102,6 +102,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.InConstantType; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion; import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport.Support; +import org.apache.hadoop.hive.ql.exec.vector.expressions.ConvertDecimal64ToDecimal; import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; @@ -4797,6 +4798,8 @@ public static Operator vectorizeSelectOperator( VectorExpression[] vectorSelectExprs = new VectorExpression[size]; int[] projectedOutputColumns = new int[size]; + int[] vectorSelectExprIndexForCol = new int[size]; + Arrays.fill(vectorSelectExprIndexForCol, -1); for (int i = 0; i < size; i++) { ExprNodeDesc expr = colList.get(i); VectorExpression ve = vContext.getVectorExpression(expr); @@ -4805,6 +4808,7 @@ public static Operator vectorizeSelectOperator( // Suppress useless evaluation. continue; } + vectorSelectExprIndexForCol[i] = index; vectorSelectExprs[index++] = ve; } if (index < size) { @@ -4817,6 +4821,12 @@ public static Operator vectorizeSelectOperator( // The following method introduces a cast if x or y is DECIMAL_64 and parent expression (x % y) is DECIMAL. try { fixDecimalDataTypePhysicalVariations(vContext, vectorSelectExprs); + for (int i = 0; i < size; i++) { + int exprIndex = vectorSelectExprIndexForCol[i]; + if (exprIndex >= 0) { + projectedOutputColumns[i] = vectorSelectExprs[exprIndex].getOutputColumnNum(); + } + } } finally { vContext.freeMarkedScratchColumns(); } @@ -4879,7 +4889,6 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto } } else { Object[] arguments; - int argumentCount = children.length + (parent.getOutputColumnNum() == -1 ? 0 : 1); // VectorCoalesce receives arguments as an array. // Need to handle it as a special case to avoid instantiation failure. if (parent instanceof VectorCoalesce) { @@ -4891,20 +4900,7 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto } arguments[1] = parent.getOutputColumnNum(); } else { - if (parent instanceof DecimalColDivideDecimalScalar) { - arguments = new Object[argumentCount + 1]; - arguments[children.length] = ((DecimalColDivideDecimalScalar) parent).getValue(); - } else { - arguments = new Object[argumentCount]; - } - for (int i = 0; i < children.length; i++) { - VectorExpression vce = children[i]; - arguments[i] = vce.getOutputColumnNum(); - } - } - // retain output column number from parent - if (parent.getOutputColumnNum() != -1) { - arguments[arguments.length - 1] = parent.getOutputColumnNum(); + arguments = buildReinstantiationArgsForDecimal64(parent, children); } // re-instantiate the parent expression with new arguments VectorExpression newParent = vContext.instantiateExpression(parent.getClass(), parent.getOutputTypeInfo(), @@ -4913,7 +4909,7 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto newParent.setOutputDataTypePhysicalVariation(parent.getOutputDataTypePhysicalVariation()); newParent.setInputTypeInfos(parent.getInputTypeInfos()); newParent.setInputDataTypePhysicalVariations(dataTypePhysicalVariations); - newParent.setChildExpressions(parent.getChildExpressions()); + newParent.setChildExpressions(children); return newParent; } } @@ -4921,6 +4917,78 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto return parent; } + /** + * Rebuild constructor arguments for a vector expression after wrapping DECIMAL_64 child + * expressions with {@link ConvertDecimal64ToDecimal}. Column reference inputs live in + * {@link VectorExpression#inputColumnNum} and are not included in childExpressions, so they + * must be preserved when re-instantiating the parent. + */ + static Object[] buildReinstantiationArgsForDecimal64(VectorExpression parent, + VectorExpression[] children) { + int[] inputColNums = extractParentInputColumnNums(parent); + replaceInputColsWithConvertedChildOutputs(inputColNums, children); + return buildParentConstructorArguments(inputColNums, parent); + } + + private static int[] extractParentInputColumnNums(VectorExpression parent) { + int inputCount = 0; + for (int col : parent.inputColumnNum) { + if (col != -1) { + inputCount++; + } + } + + int[] inputColNums = new int[inputCount]; + int idx = 0; + for (int col : parent.inputColumnNum) { + if (col != -1) { + inputColNums[idx++] = col; + } + } + return inputColNums; + } + + /** + * For each wrapped DECIMAL_64 child, replace its pre-conversion input column slot in + * {@code inputColNums} with the {@link ConvertDecimal64ToDecimal} output column. + */ + private static void replaceInputColsWithConvertedChildOutputs(int[] inputColNums, + VectorExpression[] children) { + for (VectorExpression child : children) { + int preConversionCol = getPreConversionColumnNum(child); + int convertedCol = child.getOutputColumnNum(); + for (int i = 0; i < inputColNums.length; i++) { + if (inputColNums[i] == preConversionCol) { + inputColNums[i] = convertedCol; + break; + } + } + } + } + + private static Object[] buildParentConstructorArguments(int[] inputColNums, + VectorExpression parent) { + int extraArgs = parent instanceof DecimalColDivideDecimalScalar ? 2 : 1; + Object[] arguments = new Object[inputColNums.length + extraArgs]; + for (int i = 0; i < inputColNums.length; i++) { + arguments[i] = inputColNums[i]; + } + int outputIndex = inputColNums.length; + if (parent instanceof DecimalColDivideDecimalScalar) { + arguments[outputIndex++] = ((DecimalColDivideDecimalScalar) parent).getValue(); + } + arguments[outputIndex] = parent.getOutputColumnNum(); + return arguments; + } + + /** Column to match in {@code inputColNums} before replacing with a converted child output. */ + private static int getPreConversionColumnNum(VectorExpression child) { + if (child instanceof ConvertDecimal64ToDecimal) { + return child.inputColumnNum[0]; + } + return child.getOutputColumnNum(); + } + private static void fillInPTFEvaluators( List windowsFunctions, String[] evaluatorFunctionNames, diff --git a/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q b/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q new file mode 100644 index 000000000000..97b7222061ef --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q @@ -0,0 +1,35 @@ +CREATE TABLE test1(col2 varchar(28), col3 decimal(26,6), col4 varchar(28)); +CREATE TABLE test2(col4 varchar(28)); + +INSERT INTO test1 VALUES ('abc', 1.00, 'DEF'); +INSERT INTO test2 VALUES ('DEF'); + +EXPLAIN VECTORIZATION DETAIL +SELECT + aa.int_cost +FROM + ( + SELECT + a.col4, + a.col3 * CASE WHEN a.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost + FROM test1 a + ) aa + INNER JOIN ( + SELECT col4 FROM test2 + ) bb ON trim(aa.col4) = trim(bb.col4); + +SELECT + aa.int_cost +FROM + ( + SELECT + a.col4, + a.col3 * CASE WHEN a.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost + FROM test1 a + ) aa + INNER JOIN ( + SELECT col4 FROM test2 + ) bb ON trim(aa.col4) = trim(bb.col4); + +DROP TABLE test1; +DROP TABLE test2; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out new file mode 100644 index 000000000000..6bb61e1c1026 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out @@ -0,0 +1,280 @@ +PREHOOK: query: CREATE TABLE test1(col2 varchar(28), col3 decimal(26,6), col4 varchar(28)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test1 +POSTHOOK: query: CREATE TABLE test1(col2 varchar(28), col3 decimal(26,6), col4 varchar(28)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test1 +PREHOOK: query: CREATE TABLE test2(col4 varchar(28)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test2 +POSTHOOK: query: CREATE TABLE test2(col4 varchar(28)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test2 +PREHOOK: query: INSERT INTO test1 VALUES ('abc', 1.00, 'DEF') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test1 +POSTHOOK: query: INSERT INTO test1 VALUES ('abc', 1.00, 'DEF') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test1 +POSTHOOK: Lineage: test1.col2 SCRIPT [] +POSTHOOK: Lineage: test1.col3 SCRIPT [] +POSTHOOK: Lineage: test1.col4 SCRIPT [] +PREHOOK: query: INSERT INTO test2 VALUES ('DEF') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test2 +POSTHOOK: query: INSERT INTO test2 VALUES ('DEF') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test2 +POSTHOOK: Lineage: test2.col4 SCRIPT [] +PREHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT + aa.int_cost +FROM + ( + SELECT + a.col4, + a.col3 * CASE WHEN a.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost + FROM test1 a + ) aa + INNER JOIN ( + SELECT col4 FROM test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL +SELECT + aa.int_cost +FROM + ( + SELECT + a.col4, + a.col3 * CASE WHEN a.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost + FROM test1 a + ) aa + INNER JOIN ( + SELECT col4 FROM test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + filterExpr: trim(col4) is not null (type: boolean) + Statistics: Num rows: 1 Data size: 286 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:col2:varchar(28), 1:col3:decimal(26,6), 2:col4:varchar(28), 3:ROW__ID:struct, 4:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 5:string)(children: StringTrimCol(col 2:varchar(28)) -> 5:string) + predicate: trim(col4) is not null (type: boolean) + Statistics: Num rows: 1 Data size: 286 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col4 (type: varchar(28)), (col3 * if((col2 = 'bc'), 1.77, 0.72)) (type: decimal(30,8)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [2, 8] + selectExpressions: DecimalColMultiplyDecimalColumn(col 1:decimal(26,6), col 9:decimal(3,2)/DECIMAL_64)(children: ConvertDecimal64ToDecimal(col 7:decimal(3,2)/DECIMAL_64)(children: IfExprDecimal64ScalarDecimal64Scalar(col 6:boolean, decimal64Val1 177, decimalVal1 1.77, decimal64Val2 72, decimalVal2 0.72)(children: StringGroupColEqualVarCharScalar(col 0:varchar(28), val bc) -> 6:boolean) -> 7:decimal(3,2)/DECIMAL_64) -> 9:decimal(3,2)) -> 8:decimal(30,8) + Statistics: Num rows: 1 Data size: 199 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: trim(_col0) (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: trim(_col0) (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + keyColumns: 5:string + keyExpressions: StringTrimCol(col 2:varchar(28)) -> 5:string + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 8:decimal(30,8) + Statistics: Num rows: 1 Data size: 199 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(30,8)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 3 + includeColumns: [0, 1, 2] + dataColumns: col2:varchar(28), col3:decimal(26,6), col4:varchar(28) + partitionColumnCount: 0 + scratchColumnTypeNames: [string, bigint, decimal(3,2)/DECIMAL_64, decimal(30,8), decimal(3,2)] + Map 3 + Map Operator Tree: + TableScan + alias: test2 + filterExpr: trim(col4) is not null (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:col4:varchar(28), 1:ROW__ID:struct, 2:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 3:string)(children: StringTrimCol(col 0:varchar(28)) -> 3:string) + predicate: trim(col4) is not null (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col4 (type: varchar(28)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [0] + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: trim(_col0) (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: trim(_col0) (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + keyColumns: 3:string + keyExpressions: StringTrimCol(col 0:varchar(28)) -> 3:string + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + includeColumns: [0] + dataColumns: col4:varchar(28) + partitionColumnCount: 0 + scratchColumnTypeNames: [string] + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 trim(_col0) (type: string) + 1 trim(_col0) (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: decimal(30,8)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + MergeJoin Vectorization: + enabled: false + enableConditionsNotMet: Vectorizing MergeJoin Supported IS false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT + aa.int_cost +FROM + ( + SELECT + a.col4, + a.col3 * CASE WHEN a.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost + FROM test1 a + ) aa + INNER JOIN ( + SELECT col4 FROM test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT + aa.int_cost +FROM + ( + SELECT + a.col4, + a.col3 * CASE WHEN a.col2 = 'bc' THEN 1.77 ELSE 0.72 END AS int_cost + FROM test1 a + ) aa + INNER JOIN ( + SELECT col4 FROM test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +0.72000000 +PREHOOK: query: DROP TABLE test1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test1 +PREHOOK: Output: database:default +PREHOOK: Output: default@test1 +POSTHOOK: query: DROP TABLE test1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test1 +PREHOOK: query: DROP TABLE test2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test2 +PREHOOK: Output: database:default +PREHOOK: Output: default@test2 +POSTHOOK: query: DROP TABLE test2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test2