@@ -31,15 +31,13 @@ pub enum CopyMode {
3131}
3232
3333/// Copy an Arrow Array
34- pub ( crate ) fn copy_array ( array : & dyn Array ) -> ArrayRef {
34+ pub ( crate ) fn copy_array ( array : & dyn Array ) -> Result < ArrayRef , ArrowError > {
3535 let capacity = array. len ( ) ;
3636 let data = array. to_data ( ) ;
3737
3838 let mut mutable = MutableArrayData :: new ( vec ! [ & data] , false , capacity) ;
3939
40- mutable
41- . try_extend ( 0 , 0 , capacity)
42- . expect ( "extend failed due to offset overflow" ) ;
40+ mutable. try_extend ( 0 , 0 , capacity) ?;
4341
4442 if matches ! ( array. data_type( ) , DataType :: Dictionary ( _, _) ) {
4543 let copied_dict = make_array ( mutable. freeze ( ) ) ;
@@ -52,17 +50,15 @@ pub(crate) fn copy_array(array: &dyn Array) -> ArrayRef {
5250 let data = values. to_data( ) ;
5351
5452 let mut mutable = MutableArrayData :: new( vec![ & data] , false , values. len( ) ) ;
55- mutable
56- . try_extend( 0 , 0 , values. len( ) )
57- . expect( "extend failed due to offset overflow" ) ;
53+ mutable. try_extend( 0 , 0 , values. len( ) ) ?;
5854
5955 let copied_dict = ref_copied_dict. with_values( make_array( mutable. freeze( ) ) ) ;
60- Arc :: new( copied_dict)
56+ Ok ( Arc :: new( copied_dict) )
6157 }
6258 t => unreachable!( "Should not reach here: {}" , t)
6359 )
6460 } else {
65- make_array ( mutable. freeze ( ) )
61+ Ok ( make_array ( mutable. freeze ( ) ) )
6662 }
6763}
6864
@@ -80,18 +76,100 @@ pub(crate) fn copy_or_unpack_array(
8076 let options = CastOptions :: default ( ) ;
8177 // We need to copy the array after `cast` because arrow-rs `take` kernel which is used
8278 // to unpack dictionary array might reuse the input array's null buffer.
83- Ok ( copy_array ( & cast_with_options (
84- array,
85- value_type. as_ref ( ) ,
86- & options,
87- ) ?) )
79+ copy_array ( & cast_with_options ( array, value_type. as_ref ( ) , & options) ?)
8880 }
8981 _ => {
9082 if mode == & CopyMode :: UnpackOrDeepCopy {
91- Ok ( copy_array ( array) )
83+ copy_array ( array)
9284 } else {
9385 Ok ( Arc :: clone ( array) )
9486 }
9587 }
9688 }
9789}
90+
91+ #[ cfg( test) ]
92+ mod tests {
93+ use super :: * ;
94+ use arrow:: array:: { DictionaryArray , Int32Array , Int8Array , ListViewArray , NullArray } ;
95+ use arrow:: datatypes:: { Field , Int8Type } ;
96+
97+ fn overflowing_list_view ( ) -> ArrayRef {
98+ // Overlapping views expand past i32::MAX when copied. NullArray stores only
99+ // a length, so the large logical child does not require a large allocation.
100+ Arc :: new ( ListViewArray :: new (
101+ Arc :: new ( Field :: new ( "item" , DataType :: Null , true ) ) ,
102+ vec ! [ 0_i32 , 0 ] . into ( ) ,
103+ vec ! [ i32 :: MAX , 1 ] . into ( ) ,
104+ Arc :: new ( NullArray :: new ( i32:: MAX as usize ) ) ,
105+ None ,
106+ ) )
107+ }
108+
109+ #[ test]
110+ fn copy_array_propagates_offset_overflow ( ) {
111+ let array = overflowing_list_view ( ) ;
112+ assert ! ( matches!(
113+ copy_array( array. as_ref( ) ) ,
114+ Err ( ArrowError :: InvalidArgumentError ( _) )
115+ ) ) ;
116+ assert ! ( matches!(
117+ copy_or_unpack_array( & array, & CopyMode :: UnpackOrDeepCopy ) ,
118+ Err ( ArrowError :: InvalidArgumentError ( _) )
119+ ) ) ;
120+ let cloned = copy_or_unpack_array ( & array, & CopyMode :: UnpackOrClone ) . unwrap ( ) ;
121+ assert ! ( Arc :: ptr_eq( & array, & cloned) ) ;
122+ }
123+
124+ #[ test]
125+ fn copy_dictionary_values_propagates_offset_overflow ( ) {
126+ let array =
127+ DictionaryArray :: < Int8Type > :: new ( Int8Array :: from ( vec ! [ 0 , 1 ] ) , overflowing_list_view ( ) ) ;
128+ assert ! ( matches!(
129+ copy_array( & array) ,
130+ Err ( ArrowError :: InvalidArgumentError ( _) )
131+ ) ) ;
132+ }
133+
134+ #[ test]
135+ fn copy_array_preserves_sliced_nullable_values ( ) {
136+ let source = Int32Array :: from ( vec ! [ Some ( 0 ) , Some ( 1 ) , None , Some ( 3 ) ] ) ;
137+ let array = source. slice ( 1 , 3 ) ;
138+ let copied = copy_array ( & array) . unwrap ( ) ;
139+ assert_eq ! ( copied. to_data( ) , array. to_data( ) ) ;
140+ let copied = copied. as_any ( ) . downcast_ref :: < Int32Array > ( ) . unwrap ( ) ;
141+ assert_ne ! ( copied. values( ) . as_ptr( ) , array. values( ) . as_ptr( ) ) ;
142+ }
143+
144+ #[ test]
145+ fn copy_dictionary_preserves_values_and_unpacks ( ) {
146+ let values = Arc :: new ( Int32Array :: from ( vec ! [ Some ( 10 ) , None , Some ( 30 ) ] ) ) ;
147+ let dictionary = DictionaryArray :: < Int8Type > :: new (
148+ Int8Array :: from ( vec ! [ Some ( 2 ) , None , Some ( 0 ) , Some ( 1 ) ] ) ,
149+ Arc :: clone ( & values) as ArrayRef ,
150+ ) ;
151+ let copied = copy_array ( & dictionary) . unwrap ( ) ;
152+ assert_eq ! ( copied. to_data( ) , dictionary. to_data( ) ) ;
153+ let copied = copied
154+ . as_any ( )
155+ . downcast_ref :: < DictionaryArray < Int8Type > > ( )
156+ . unwrap ( ) ;
157+ assert_ne ! (
158+ copied. keys( ) . values( ) . as_ptr( ) ,
159+ dictionary. keys( ) . values( ) . as_ptr( )
160+ ) ;
161+ let copied_values = copied
162+ . values ( )
163+ . as_any ( )
164+ . downcast_ref :: < Int32Array > ( )
165+ . unwrap ( ) ;
166+ assert_ne ! ( copied_values. values( ) . as_ptr( ) , values. values( ) . as_ptr( ) ) ;
167+
168+ let array: ArrayRef = Arc :: new ( dictionary) ;
169+ for mode in [ CopyMode :: UnpackOrDeepCopy , CopyMode :: UnpackOrClone ] {
170+ let unpacked = copy_or_unpack_array ( & array, & mode) . unwrap ( ) ;
171+ let expected = Int32Array :: from ( vec ! [ Some ( 30 ) , None , Some ( 10 ) , None ] ) ;
172+ assert_eq ! ( unpacked. to_data( ) , expected. to_data( ) ) ;
173+ }
174+ }
175+ }
0 commit comments