diff --git a/python/pyspark/sql/tests/test_udtf.py b/python/pyspark/sql/tests/test_udtf.py index a272cee4c7b57..2b30e41b30db2 100644 --- a/python/pyspark/sql/tests/test_udtf.py +++ b/python/pyspark/sql/tests/test_udtf.py @@ -1999,6 +1999,25 @@ def eval(self): with self.assertRaisesRegex(AnalysisException, "Failed to analyze."): func().collect() + def test_udtf_analyze_traceback_with_locals(self): + with self.sql_conf({"spark.sql.execution.pyspark.udf.tracebackWithLocals.enabled": True}): + + class TestUDTF: + @staticmethod + def analyze() -> AnalyzeResult: + local_marker = 1 + if local_marker: + raise ValueError("boom") + return AnalyzeResult(StructType().add("x", StringType())) + + def eval(self): + yield ("x",) + + func = udtf(TestUDTF) + + with self.assertRaisesRegex(AnalysisException, "local_marker = 1"): + func().collect() + def test_udtf_with_analyze_null_literal(self): class TestUDTF: @staticmethod diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala index 80887dd469d69..063a1c04eb6ce 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/PythonPlannerRunner.scala @@ -91,6 +91,9 @@ abstract class PythonPlannerRunner[T](func: PythonFunction) extends Logging { if (simplifiedTraceback) { envVars.put("SPARK_SIMPLIFIED_TRACEBACK", "1") } + if (tracebackWithLocals) { + envVars.put("SPARK_TRACEBACK_WITH_LOCALS", "1") + } workerMemoryMb.foreach { memoryMb => envVars.put("PYSPARK_PLANNER_MEMORY_MB", memoryMb.toString) }