diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json index 6c7612da286db..07f47a0c91009 100644 --- a/python/pyspark/errors/error-conditions.json +++ b/python/pyspark/errors/error-conditions.json @@ -1208,11 +1208,6 @@ "Value for `` must be a non-empty string, got ''." ] }, - "VALUE_NOT_PEARSON": { - "message": [ - "Value for `` only supports 'pearson', got ''." - ] - }, "VALUE_NOT_PLAIN_COLUMN_REFERENCE": { "message": [ "Value `` in `` should be a plain column reference such as `df.col` or `col('column')`." diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py index ca4b1f17de2a2..9599fad3bcdaa 100644 --- a/python/pyspark/sql/classic/dataframe.py +++ b/python/pyspark/sql/classic/dataframe.py @@ -1693,8 +1693,8 @@ def corr(self, col1: str, col2: str, method: Optional[str] = None) -> float: method = "pearson" if not method == "pearson": raise PySparkValueError( - errorClass="VALUE_NOT_PEARSON", - messageParameters={"arg_name": "method", "arg_value": method}, + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, ) return self._jdf.stat().corr(col1, col2, method) diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py index 64277d1b35b64..99ee077099370 100644 --- a/python/pyspark/sql/connect/dataframe.py +++ b/python/pyspark/sql/connect/dataframe.py @@ -1663,8 +1663,8 @@ def corr(self, col1: str, col2: str, method: Optional[str] = None) -> float: method = "pearson" if not method == "pearson": raise PySparkValueError( - errorClass="VALUE_NOT_PEARSON", - messageParameters={"arg_name": "method", "arg_value": method}, + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, ) table, _ = DataFrame( plan.StatCorr(child=self._plan, col1=col1, col2=col2, method=method), diff --git a/python/pyspark/sql/tests/connect/test_connect_stat.py b/python/pyspark/sql/tests/connect/test_connect_stat.py index c5f6868996374..3d05cb7b4bb5c 100644 --- a/python/pyspark/sql/tests/connect/test_connect_stat.py +++ b/python/pyspark/sql/tests/connect/test_connect_stat.py @@ -214,13 +214,14 @@ def test_stat_corr(self): "arg_type": "int", }, ) - with self.assertRaises(ValueError) as context: - (self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3", "spearman"),) - self.assertTrue( - "Currently only the calculation of the Pearson Correlation " - + "coefficient is supported." - in str(context.exception) - ) + with self.assertRaises(PySparkValueError) as pe: + self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3", "spearman") + + self.check_error( + exception=pe.exception, + errorClass="VALUE_NOT_ALLOWED", + messageParameters={"arg_name": "method", "allowed_values": "['pearson']"}, + ) def test_stat_approx_quantile(self): # SPARK-41069: Test the stat.approxQuantile method