From c2c10e58771e8b4fc0460c18b606760eb4b0e09e Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 20 Jul 2026 06:00:52 +0000 Subject: [PATCH 1/2] [WIP][PYTHON] Use VALUE_NOT_ALLOWED for corr --- python/pyspark/errors/error-conditions.json | 5 ----- python/pyspark/sql/classic/dataframe.py | 4 ++-- python/pyspark/sql/connect/dataframe.py | 4 ++-- .../sql/tests/connect/test_connect_stat.py | 17 +++++++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json index 6c7612da286db..07f47a0c91009 100644 --- a/python/pyspark/errors/error-conditions.json +++ b/python/pyspark/errors/error-conditions.json @@ -1208,11 +1208,6 @@ "Value for `` must be a non-empty string, got ''." ] }, - "VALUE_NOT_PEARSON": { - "message": [ - "Value for `` only supports 'pearson', got ''." - ] - }, "VALUE_NOT_PLAIN_COLUMN_REFERENCE": { "message": [ "Value `` in `` should be a plain column reference such as `df.col` or `col('column')`." diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py index ca4b1f17de2a2..9599fad3bcdaa 100644 --- a/python/pyspark/sql/classic/dataframe.py +++ b/python/pyspark/sql/classic/dataframe.py @@ -1693,8 +1693,8 @@ def corr(self, col1: str, col2: str, method: Optional[str] = None) -> float: method = "pearson" if not method == "pearson": raise PySparkValueError( - errorClass="VALUE_NOT_PEARSON", - messageParameters={"arg_name": "method", "arg_value": method}, + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, ) return self._jdf.stat().corr(col1, col2, method) diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py index 64277d1b35b64..99ee077099370 100644 --- a/python/pyspark/sql/connect/dataframe.py +++ b/python/pyspark/sql/connect/dataframe.py @@ -1663,8 +1663,8 @@ def corr(self, col1: str, col2: str, method: Optional[str] = None) -> float: method = "pearson" if not method == "pearson": raise PySparkValueError( - errorClass="VALUE_NOT_PEARSON", - messageParameters={"arg_name": "method", "arg_value": method}, + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, ) table, _ = DataFrame( plan.StatCorr(child=self._plan, col1=col1, col2=col2, method=method), diff --git a/python/pyspark/sql/tests/connect/test_connect_stat.py b/python/pyspark/sql/tests/connect/test_connect_stat.py index c5f6868996374..3a48c110509f2 100644 --- a/python/pyspark/sql/tests/connect/test_connect_stat.py +++ b/python/pyspark/sql/tests/connect/test_connect_stat.py @@ -214,12 +214,17 @@ def test_stat_corr(self): "arg_type": "int", }, ) - with self.assertRaises(ValueError) as context: - (self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3", "spearman"),) - self.assertTrue( - "Currently only the calculation of the Pearson Correlation " - + "coefficient is supported." - in str(context.exception) + for df in [ + self.connect.read.table(self.tbl_name2), + self.spark.read.table(self.tbl_name2), + ]: + with self.assertRaises(PySparkValueError) as pe: + df.stat.corr("col1", "col3", "spearman") + + self.check_error( + exception=pe.exception, + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, ) def test_stat_approx_quantile(self): From ec9c86e3b31a3406b9c746a3f4d15b86dcdf6af1 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Mon, 20 Jul 2026 06:40:16 +0000 Subject: [PATCH 2/2] [WIP][PYTHON] Simplify corr error test --- .../sql/tests/connect/test_connect_stat.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/python/pyspark/sql/tests/connect/test_connect_stat.py b/python/pyspark/sql/tests/connect/test_connect_stat.py index 3a48c110509f2..3d05cb7b4bb5c 100644 --- a/python/pyspark/sql/tests/connect/test_connect_stat.py +++ b/python/pyspark/sql/tests/connect/test_connect_stat.py @@ -214,18 +214,14 @@ def test_stat_corr(self): "arg_type": "int", }, ) - for df in [ - self.connect.read.table(self.tbl_name2), - self.spark.read.table(self.tbl_name2), - ]: - with self.assertRaises(PySparkValueError) as pe: - df.stat.corr("col1", "col3", "spearman") - - self.check_error( - exception=pe.exception, - errorClass="VALUE_NOT_ALLOWED", - messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, - ) + with self.assertRaises(PySparkValueError) as pe: + self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3", "spearman") + + self.check_error( + exception=pe.exception, + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, + ) def test_stat_approx_quantile(self): # SPARK-41069: Test the stat.approxQuantile method