From ead9c746689c202e2e9c5000c3f8d37220b5f19e Mon Sep 17 00:00:00 2001 From: Xinrong Meng Date: Tue, 22 Jul 2025 17:47:25 -0700 Subject: [PATCH 1/3] 'abc' as int --- python/pyspark/pandas/data_type_ops/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/pyspark/pandas/data_type_ops/base.py b/python/pyspark/pandas/data_type_ops/base.py index b4a6b1abbcaf9..3cde6a6d62781 100644 --- a/python/pyspark/pandas/data_type_ops/base.py +++ b/python/pyspark/pandas/data_type_ops/base.py @@ -52,6 +52,7 @@ extension_object_dtypes_available, spark_type_to_pandas_dtype, ) +from pyspark.pandas.utils import is_ansi_mode_enabled if extension_dtypes_available: from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype @@ -189,7 +190,10 @@ def _as_other_type(index_ops: IndexOpsLike, dtype: Dtype, spark_type: DataType) ) assert not need_pre_process, "Pre-processing is needed before the type casting." - scol = index_ops.spark.column.cast(spark_type) + if is_ansi_mode_enabled(index_ops._internal.spark_frame.sparkSession): + scol = index_ops.spark.column.try_cast(spark_type) + else: + scol = index_ops.spark.column.cast(spark_type) return index_ops._with_new_scol(scol, field=InternalField(dtype=dtype)) From 902784ee5f757a8f893c2fc9cd2ed73fd4da9614 Mon Sep 17 00:00:00 2001 From: Xinrong Meng Date: Tue, 22 Jul 2025 17:56:38 -0700 Subject: [PATCH 2/3] test --- python/pyspark/pandas/tests/series/test_as_type.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyspark/pandas/tests/series/test_as_type.py b/python/pyspark/pandas/tests/series/test_as_type.py index dfd66ee05d382..8c8a45ebe3fe6 100644 --- a/python/pyspark/pandas/tests/series/test_as_type.py +++ b/python/pyspark/pandas/tests/series/test_as_type.py @@ -53,6 +53,7 @@ def test_astype(self): self.assert_eq(psser.astype(bool), pser.astype(bool)) self.assert_eq(psser.astype(str), pser.astype(str)) self.assert_eq(psser.str.strip().astype(bool), pser.str.strip().astype(bool)) + self.assert_eq(ps.Series(["abc"]).astype(int), pd.Series([float('nan')])) if extension_object_dtypes_available: from pandas import StringDtype From 28ed3e45e3c2d92a5ca24a19cffe6598c7f8e8a5 Mon Sep 17 00:00:00 2001 From: Xinrong Meng Date: Wed, 23 Jul 2025 17:39:47 -0700 Subject: [PATCH 3/3] lint --- python/pyspark/pandas/tests/series/test_as_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/pandas/tests/series/test_as_type.py b/python/pyspark/pandas/tests/series/test_as_type.py index 8c8a45ebe3fe6..91d6a7f3b3013 100644 --- a/python/pyspark/pandas/tests/series/test_as_type.py +++ b/python/pyspark/pandas/tests/series/test_as_type.py @@ -53,7 +53,7 @@ def test_astype(self): self.assert_eq(psser.astype(bool), pser.astype(bool)) self.assert_eq(psser.astype(str), pser.astype(str)) self.assert_eq(psser.str.strip().astype(bool), pser.str.strip().astype(bool)) - self.assert_eq(ps.Series(["abc"]).astype(int), pd.Series([float('nan')])) + self.assert_eq(ps.Series(["abc"]).astype(int), pd.Series([float("nan")])) if extension_object_dtypes_available: from pandas import StringDtype