Skip to content

Commit 7c5cf92

Browse files
seddonm1alamb
authored andcommitted
ARROW-11561: [Rust][DataFusion] Add Send + Sync to MemTable::load
This PR adds Send + Sync to the MemTable::load method to allow implementation of a `persist` method like Spark's Dataframe in an async function. Closes #9448 from seddonm1/send-sync Authored-by: Mike Seddon <seddonm1@gmail.com> Signed-off-by: Andrew Lamb <andrew@nerdnetworks.org>
1 parent 5e3fcfa commit 7c5cf92

3 files changed

Lines changed: 5 additions & 8 deletions

File tree

rust/benchmarks/src/bin/tpch.rs

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -142,12 +142,9 @@ async fn benchmark(opt: BenchmarkOpt) -> Result<Vec<arrow::record_batch::RecordB
142142
println!("Loading table '{}' into memory", table);
143143
let start = Instant::now();
144144

145-
let memtable = MemTable::load(
146-
table_provider.as_ref(),
147-
opt.batch_size,
148-
Some(opt.partitions),
149-
)
150-
.await?;
145+
let memtable =
146+
MemTable::load(table_provider, opt.batch_size, Some(opt.partitions))
147+
.await?;
151148
println!(
152149
"Loaded table '{}' into memory in {} ms",
153150
table,

rust/datafusion/benches/sort_limit_query_sql.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ fn create_context() -> Arc<Mutex<ExecutionContext>> {
7474
let partitions = 16;
7575

7676
rt.block_on(async {
77-
let mem_table = MemTable::load(&csv, 16 * 1024, Some(partitions))
77+
let mem_table = MemTable::load(Box::new(csv), 16 * 1024, Some(partitions))
7878
.await
7979
.unwrap();
8080

rust/datafusion/src/datasource/memory.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ impl MemTable {
107107

108108
/// Create a mem table by reading from another data source
109109
pub async fn load(
110-
t: &dyn TableProvider,
110+
t: Box<dyn TableProvider + Send + Sync>,
111111
batch_size: usize,
112112
output_partitions: Option<usize>,
113113
) -> Result<Self> {

0 commit comments

Comments
 (0)