From 9ca009903a14b61a4ab4455dad2e042e51f4b0c3 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 4 Apr 2024 09:39:59 -0700 Subject: [PATCH 1/2] [SPARK-46812][PYTHON][TESTS][FOLLOWUP] Check should_test_connect and pyarrow to skip tests --- .../resource/tests/test_connect_resources.py | 7 +++++-- python/pyspark/resource/tests/test_resources.py | 13 +++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python/pyspark/resource/tests/test_connect_resources.py b/python/pyspark/resource/tests/test_connect_resources.py index 40c68029a1535..1529a33cb0ad0 100644 --- a/python/pyspark/resource/tests/test_connect_resources.py +++ b/python/pyspark/resource/tests/test_connect_resources.py @@ -18,10 +18,13 @@ from pyspark.resource import ResourceProfileBuilder, TaskResourceRequests, ExecutorResourceRequests from pyspark.sql import SparkSession -from pyspark.testing.sqlutils import have_pandas, pandas_requirement_message +from pyspark.testing.connectutils import ( + should_test_connect, + connect_requirement_message, +) -@unittest.skipIf(not have_pandas, pandas_requirement_message) +@unittest.skipIf(not should_test_connect, connect_requirement_message) class ResourceProfileTests(unittest.TestCase): def test_profile_before_sc_for_connect(self): rpb = ResourceProfileBuilder() diff --git a/python/pyspark/resource/tests/test_resources.py b/python/pyspark/resource/tests/test_resources.py index 6f61d5af2d926..ed0b938532c21 100644 --- a/python/pyspark/resource/tests/test_resources.py +++ b/python/pyspark/resource/tests/test_resources.py @@ -15,10 +15,16 @@ # limitations under the License. # import unittest +from typing import cast from pyspark.resource import ExecutorResourceRequests, ResourceProfileBuilder, TaskResourceRequests from pyspark.sql import SparkSession -from pyspark.testing.sqlutils import have_pandas, pandas_requirement_message +from pyspark.testing.sqlutils import ( + have_pandas, + have_pyarrow, + pandas_requirement_message, + pyarrow_requirement_message +) class ResourceProfileTests(unittest.TestCase): @@ -72,7 +78,10 @@ def assert_request_contents(exec_reqs, task_reqs): assert_request_contents(rp3.executorResources, rp3.taskResources) sc.stop() - @unittest.skipIf(not have_pandas, pandas_requirement_message) + @unittest.skipIf( + not have_pandas or not have_pyarrow, + cast(str, pandas_requirement_message or pyarrow_requirement_message), + ) def test_profile_before_sc_for_sql(self): rpb = ResourceProfileBuilder() treqs = TaskResourceRequests().cpus(2) From bbca08e1ffbb3e03b30f6e96885bcdbc296d4b88 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 4 Apr 2024 09:47:08 -0700 Subject: [PATCH 2/2] reformat python --- python/pyspark/resource/tests/test_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/resource/tests/test_resources.py b/python/pyspark/resource/tests/test_resources.py index ed0b938532c21..e29a77ed36dda 100644 --- a/python/pyspark/resource/tests/test_resources.py +++ b/python/pyspark/resource/tests/test_resources.py @@ -23,7 +23,7 @@ have_pandas, have_pyarrow, pandas_requirement_message, - pyarrow_requirement_message + pyarrow_requirement_message, )