From 094c09742c5f9487862eac25519e2ebbeec55985 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 9 Dec 2025 16:19:50 -0800 Subject: [PATCH 1/3] Enable ruff on our CI and lint-python --- dev/lint-python | 1 + dev/spark-test-image/lint/Dockerfile | 1 + python/pyspark/sql/tests/arrow/test_arrow_udf_typehints.py | 2 +- python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py | 2 +- .../pandas/test_pandas_udf_typehints_with_future_annotations.py | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/lint-python b/dev/lint-python index fab7ad9ebe26..3b64671c5cde 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -79,6 +79,7 @@ if [[ -z "$COMPILE_TEST$BLACK_TEST$PYSPARK_CUSTOM_ERRORS_CHECK_TEST$FLAKE8_TEST$ BLACK_TEST=true PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true FLAKE8_TEST=true + RUFF_TEST=true MYPY_TEST=true MYPY_EXAMPLES_TEST=true MYPY_DATA_TEST=true diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile index 6ab571bf35d6..48076ce5eb77 100644 --- a/dev/spark-test-image/lint/Dockerfile +++ b/dev/spark-test-image/lint/Dockerfile @@ -80,6 +80,7 @@ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 RUN python3.11 -m pip install \ 'black==23.12.1' \ 'flake8==3.9.0' \ + 'ruff' \ 'googleapis-common-protos-stubs==2.2.0' \ 'grpc-stubs==1.24.11' \ 'grpcio-status==1.76.0' \ diff --git a/python/pyspark/sql/tests/arrow/test_arrow_udf_typehints.py b/python/pyspark/sql/tests/arrow/test_arrow_udf_typehints.py index a117a049952c..5136cadf7ab8 100644 --- a/python/pyspark/sql/tests/arrow/test_arrow_udf_typehints.py +++ b/python/pyspark/sql/tests/arrow/test_arrow_udf_typehints.py @@ -461,7 +461,7 @@ def multiply_pandas(a: pd.Series, b: pd.Series) -> pd.Series: if __name__ == "__main__": - from pyspark.sql.tests.arrow.test_arrow_udf_typehints import * # noqa: #F401 + from pyspark.sql.tests.arrow.test_arrow_udf_typehints import * # noqa: F401 try: import xmlrunner diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py index 7e9f22bb94bb..d4c1cf5d62ff 100644 --- a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py +++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py @@ -444,7 +444,7 @@ def multiply_arrow(a: pa.Array, b: pa.Array) -> pa.Array: if __name__ == "__main__": - from pyspark.sql.tests.pandas.test_pandas_udf_typehints import * # noqa: #F401 + from pyspark.sql.tests.pandas.test_pandas_udf_typehints import * # noqa: F401 try: import xmlrunner diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py index 88cc4b81778a..9454be108b62 100644 --- a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py +++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py @@ -367,7 +367,7 @@ def func(col: "Union[pd.Series, pd.DataFrame]", *, col2: "pd.DataFrame") -> "pd. if __name__ == "__main__": - from pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations import * # noqa: #F401 + from pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations import * # noqa: F401 try: import xmlrunner From b810e7840d58b65db0125101cde44dc47407f5bf Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 9 Dec 2025 16:37:29 -0800 Subject: [PATCH 2/3] Pin ruff version --- dev/spark-test-image/lint/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile index 48076ce5eb77..4984f56fc763 100644 --- a/dev/spark-test-image/lint/Dockerfile +++ b/dev/spark-test-image/lint/Dockerfile @@ -80,7 +80,7 @@ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 RUN python3.11 -m pip install \ 'black==23.12.1' \ 'flake8==3.9.0' \ - 'ruff' \ + 'ruff==0.14.8' \ 'googleapis-common-protos-stubs==2.2.0' \ 'grpc-stubs==1.24.11' \ 'grpcio-status==1.76.0' \ From 910d03c9d061cf19383c197b592702008f5b84fd Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 9 Dec 2025 18:46:27 -0800 Subject: [PATCH 3/3] Add E501 to ignore list --- .../test_pandas_udf_typehints_with_future_annotations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py index 9454be108b62..d2e5dd167316 100644 --- a/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py +++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py @@ -367,7 +367,8 @@ def func(col: "Union[pd.Series, pd.DataFrame]", *, col2: "pd.DataFrame") -> "pd. if __name__ == "__main__": - from pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations import * # noqa: F401 + # E501: line too long + from pyspark.sql.tests.pandas.test_pandas_udf_typehints_with_future_annotations import * # noqa: F401, E501 try: import xmlrunner