From 62cc971cafeca50371f043a8adeceb3a10500b67 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 12 Oct 2026 22:44:15 -0700 Subject: [PATCH] [SPARK-60211][PYTHON][4.1] Fix applyInPandasWithState failure with PyArrow 26 due to float slice offsets Co-authored-by: Isaac --- python/pyspark/sql/pandas/serializers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py index 0ccdffdcd1f86..6d8503d354176 100644 --- a/python/pyspark/sql/pandas/serializers.py +++ b/python/pyspark/sql/pandas/serializers.py @@ -1573,8 +1573,11 @@ def gen_data_and_state(batches): # no more data with grouping key + state break - data_start_offset = state_info_col["startOffset"] - num_data_rows = state_info_col["numRows"] + # The int fields of the state struct are converted to float when the batch + # contains padding (null) state rows. PyArrow >= 26 rejects float offsets + # in slice(), so convert them back to int. + data_start_offset = int(state_info_col["startOffset"]) + num_data_rows = int(state_info_col["numRows"]) is_last_chunk = state_info_col["isLastChunk"] if state_for_current_group: