Skip to content

Commit 8a67fe0

Browse files
authored
fix: propagate Arrow array copy errors (#5747)
1 parent bc74cc7 commit 8a67fe0

1 file changed

Lines changed: 93 additions & 15 deletions

File tree

  • native/core/src/execution/operators

native/core/src/execution/operators/copy.rs

Lines changed: 93 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,13 @@ pub enum CopyMode {
3131
}
3232

3333
/// Copy an Arrow Array
34-
pub(crate) fn copy_array(array: &dyn Array) -> ArrayRef {
34+
pub(crate) fn copy_array(array: &dyn Array) -> Result<ArrayRef, ArrowError> {
3535
let capacity = array.len();
3636
let data = array.to_data();
3737

3838
let mut mutable = MutableArrayData::new(vec![&data], false, capacity);
3939

40-
mutable
41-
.try_extend(0, 0, capacity)
42-
.expect("extend failed due to offset overflow");
40+
mutable.try_extend(0, 0, capacity)?;
4341

4442
if matches!(array.data_type(), DataType::Dictionary(_, _)) {
4543
let copied_dict = make_array(mutable.freeze());
@@ -52,17 +50,15 @@ pub(crate) fn copy_array(array: &dyn Array) -> ArrayRef {
5250
let data = values.to_data();
5351

5452
let mut mutable = MutableArrayData::new(vec![&data], false, values.len());
55-
mutable
56-
.try_extend(0, 0, values.len())
57-
.expect("extend failed due to offset overflow");
53+
mutable.try_extend(0, 0, values.len())?;
5854

5955
let copied_dict = ref_copied_dict.with_values(make_array(mutable.freeze()));
60-
Arc::new(copied_dict)
56+
Ok(Arc::new(copied_dict))
6157
}
6258
t => unreachable!("Should not reach here: {}", t)
6359
)
6460
} else {
65-
make_array(mutable.freeze())
61+
Ok(make_array(mutable.freeze()))
6662
}
6763
}
6864

@@ -80,18 +76,100 @@ pub(crate) fn copy_or_unpack_array(
8076
let options = CastOptions::default();
8177
// We need to copy the array after `cast` because arrow-rs `take` kernel which is used
8278
// to unpack dictionary array might reuse the input array's null buffer.
83-
Ok(copy_array(&cast_with_options(
84-
array,
85-
value_type.as_ref(),
86-
&options,
87-
)?))
79+
copy_array(&cast_with_options(array, value_type.as_ref(), &options)?)
8880
}
8981
_ => {
9082
if mode == &CopyMode::UnpackOrDeepCopy {
91-
Ok(copy_array(array))
83+
copy_array(array)
9284
} else {
9385
Ok(Arc::clone(array))
9486
}
9587
}
9688
}
9789
}
90+
91+
#[cfg(test)]
92+
mod tests {
93+
use super::*;
94+
use arrow::array::{DictionaryArray, Int32Array, Int8Array, ListViewArray, NullArray};
95+
use arrow::datatypes::{Field, Int8Type};
96+
97+
fn overflowing_list_view() -> ArrayRef {
98+
// Overlapping views expand past i32::MAX when copied. NullArray stores only
99+
// a length, so the large logical child does not require a large allocation.
100+
Arc::new(ListViewArray::new(
101+
Arc::new(Field::new("item", DataType::Null, true)),
102+
vec![0_i32, 0].into(),
103+
vec![i32::MAX, 1].into(),
104+
Arc::new(NullArray::new(i32::MAX as usize)),
105+
None,
106+
))
107+
}
108+
109+
#[test]
110+
fn copy_array_propagates_offset_overflow() {
111+
let array = overflowing_list_view();
112+
assert!(matches!(
113+
copy_array(array.as_ref()),
114+
Err(ArrowError::InvalidArgumentError(_))
115+
));
116+
assert!(matches!(
117+
copy_or_unpack_array(&array, &CopyMode::UnpackOrDeepCopy),
118+
Err(ArrowError::InvalidArgumentError(_))
119+
));
120+
let cloned = copy_or_unpack_array(&array, &CopyMode::UnpackOrClone).unwrap();
121+
assert!(Arc::ptr_eq(&array, &cloned));
122+
}
123+
124+
#[test]
125+
fn copy_dictionary_values_propagates_offset_overflow() {
126+
let array =
127+
DictionaryArray::<Int8Type>::new(Int8Array::from(vec![0, 1]), overflowing_list_view());
128+
assert!(matches!(
129+
copy_array(&array),
130+
Err(ArrowError::InvalidArgumentError(_))
131+
));
132+
}
133+
134+
#[test]
135+
fn copy_array_preserves_sliced_nullable_values() {
136+
let source = Int32Array::from(vec![Some(0), Some(1), None, Some(3)]);
137+
let array = source.slice(1, 3);
138+
let copied = copy_array(&array).unwrap();
139+
assert_eq!(copied.to_data(), array.to_data());
140+
let copied = copied.as_any().downcast_ref::<Int32Array>().unwrap();
141+
assert_ne!(copied.values().as_ptr(), array.values().as_ptr());
142+
}
143+
144+
#[test]
145+
fn copy_dictionary_preserves_values_and_unpacks() {
146+
let values = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)]));
147+
let dictionary = DictionaryArray::<Int8Type>::new(
148+
Int8Array::from(vec![Some(2), None, Some(0), Some(1)]),
149+
Arc::clone(&values) as ArrayRef,
150+
);
151+
let copied = copy_array(&dictionary).unwrap();
152+
assert_eq!(copied.to_data(), dictionary.to_data());
153+
let copied = copied
154+
.as_any()
155+
.downcast_ref::<DictionaryArray<Int8Type>>()
156+
.unwrap();
157+
assert_ne!(
158+
copied.keys().values().as_ptr(),
159+
dictionary.keys().values().as_ptr()
160+
);
161+
let copied_values = copied
162+
.values()
163+
.as_any()
164+
.downcast_ref::<Int32Array>()
165+
.unwrap();
166+
assert_ne!(copied_values.values().as_ptr(), values.values().as_ptr());
167+
168+
let array: ArrayRef = Arc::new(dictionary);
169+
for mode in [CopyMode::UnpackOrDeepCopy, CopyMode::UnpackOrClone] {
170+
let unpacked = copy_or_unpack_array(&array, &mode).unwrap();
171+
let expected = Int32Array::from(vec![Some(30), None, Some(10), None]);
172+
assert_eq!(unpacked.to_data(), expected.to_data());
173+
}
174+
}
175+
}

0 commit comments

Comments
 (0)