Skip to content

[VL] Upgrade velox to 2023.3.29 #1231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhejiangxiaomai merged 1 commit into from
Apr 1, 2023
Merged

[VL] Upgrade velox to 2023.3.29 #1231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions cpp/velox/benchmarks/ColumnarToRowBenchmark.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ class GoogleBenchmarkColumnarToRow {
ArrowArray arrowArray;
ArrowSchema arrowSchema;
ASSERT_NOT_OK(arrow::ExportRecordBatch(rb, &arrowArray, &arrowSchema));
return velox::importFromArrowAsOwner(arrowSchema, arrowArray, gluten::GetDefaultWrappedVeloxMemoryPool());
return velox::importFromArrowAsOwner(arrowSchema, arrowArray, gluten::GetDefaultWrappedVeloxMemoryPool().get());
}

protected:
Expand Down Expand Up @@ -153,10 +153,11 @@ class GoogleBenchmarkColumnarToRow_CacheScan_Benchmark : public GoogleBenchmarkC

auto arrowPool = GetDefaultWrappedArrowMemoryPool();
auto veloxPool = AsWrappedVeloxMemoryPool(DefaultMemoryAllocator().get());
auto ctxPool = veloxPool->addChild("cache_scan", velox::memory::MemoryPool::Kind::kLeaf);
for (auto _ : state) {
for (const auto& vector : vectors) {
auto columnarToRowConverter = std::make_shared<gluten::VeloxColumnarToRowConverter>(
std::dynamic_pointer_cast<velox::RowVector>(vector), arrowPool, veloxPool);
std::dynamic_pointer_cast<velox::RowVector>(vector), arrowPool, ctxPool);
TIME_NANO_OR_THROW(init_time, columnarToRowConverter->Init());
TIME_NANO_OR_THROW(write_time, columnarToRowConverter->Write());
}
Expand Down Expand Up @@ -203,6 +204,7 @@ class GoogleBenchmarkColumnarToRow_IterateScan_Benchmark : public GoogleBenchmar

auto arrowPool = GetDefaultWrappedArrowMemoryPool();
auto veloxPool = AsWrappedVeloxMemoryPool(DefaultMemoryAllocator().get());
auto ctxPool = veloxPool->addChild("iterate_scan", velox::memory::MemoryPool::Kind::kLeaf);
for (auto _ : state) {
ASSERT_NOT_OK(parquet_reader->GetRecordBatchReader(row_group_indices, column_indices, &record_batch_reader));
TIME_NANO_OR_THROW(elapse_read, record_batch_reader->ReadNext(&record_batch));
Expand All @@ -211,7 +213,7 @@ class GoogleBenchmarkColumnarToRow_IterateScan_Benchmark : public GoogleBenchmar
num_rows += record_batch->num_rows();
auto vector = recordBatch2RowVector(*record_batch);
auto columnarToRowConverter = std::make_shared<gluten::VeloxColumnarToRowConverter>(
std::dynamic_pointer_cast<velox::RowVector>(vector), arrowPool, veloxPool);
std::dynamic_pointer_cast<velox::RowVector>(vector), arrowPool, ctxPool);
TIME_NANO_OR_THROW(init_time, columnarToRowConverter->Init());
TIME_NANO_OR_THROW(write_time, columnarToRowConverter->Write());
TIME_NANO_OR_THROW(elapse_read, record_batch_reader->ReadNext(&record_batch));
Expand Down
14 changes: 7 additions & 7 deletions cpp/velox/compute/VeloxBackend.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <filesystem>

#include "VeloxBackend.h"
#include <filesystem>

#include "ArrowTypeUtils.h"
#include "VeloxBridge.h"
Expand Down Expand Up @@ -269,8 +269,7 @@ std::shared_ptr<ResultIterator> VeloxBackend::GetResultIterator(
getInfoAndIds(subVeloxPlanConverter_->splitInfos(), veloxPlan_->leafPlanNodeIds(), scanInfos, scanIds, streamIds);

auto veloxPool = AsWrappedVeloxMemoryPool(allocator);
auto ctxPool = veloxPool->addChild("ctx_root");
ctxPool->setMemoryUsageTracker(memUsageTracker_->addChild());
auto ctxPool = veloxPool->addChild("result_iterator_spill", velox::memory::MemoryPool::Kind::kAggregate);
if (scanInfos.size() == 0) {
// Source node is not required.
auto wholestageIter =
Expand Down Expand Up @@ -298,8 +297,7 @@ std::shared_ptr<ResultIterator> VeloxBackend::GetResultIterator(
getInfoAndIds(subVeloxPlanConverter_->splitInfos(), veloxPlan_->leafPlanNodeIds(), scanInfos, scanIds, streamIds);

auto veloxPool = AsWrappedVeloxMemoryPool(allocator);
auto ctxPool = veloxPool->addChild("ctx_root");
ctxPool->setMemoryUsageTracker(memUsageTracker_->addChild());
auto ctxPool = veloxPool->addChild("result_iterator", velox::memory::MemoryPool::Kind::kLeaf);
auto wholestageIter = std::make_unique<WholeStageResultIteratorFirstStage>(
ctxPool, veloxPlan_, scanIds, setScanInfos, streamIds, "/tmp/test-spill", confMap_);
return std::make_shared<ResultIterator>(std::move(wholestageIter), shared_from_this());
Expand All @@ -312,7 +310,8 @@ arrow::Result<std::shared_ptr<ColumnarToRowConverter>> VeloxBackend::getColumnar
if (veloxBatch != nullptr) {
auto arrowPool = AsWrappedArrowMemoryPool(allocator);
auto veloxPool = AsWrappedVeloxMemoryPool(allocator);
return std::make_shared<VeloxColumnarToRowConverter>(veloxBatch->getFlattenedRowVector(), arrowPool, veloxPool);
auto ctxVeloxPool = veloxPool->addChild("columnar_to_row_velox", velox::memory::MemoryPool::Kind::kLeaf);
return std::make_shared<VeloxColumnarToRowConverter>(veloxBatch->getFlattenedRowVector(), arrowPool, ctxVeloxPool);
} else {
return Backend::getColumnar2RowConverter(allocator, cb);
}
Expand Down Expand Up @@ -349,7 +348,8 @@ std::shared_ptr<facebook::velox::memory::MemoryUsageTracker> VeloxBackend::getMe

void VeloxBackend::cacheOutputSchema(const std::shared_ptr<const velox::core::PlanNode>& planNode) {
ArrowSchema arrowSchema{};
exportToArrow(velox::BaseVector::create(planNode->outputType(), 0, GetDefaultWrappedVeloxMemoryPool()), arrowSchema);
exportToArrow(
velox::BaseVector::create(planNode->outputType(), 0, GetDefaultWrappedVeloxMemoryPool().get()), arrowSchema);
GLUTEN_ASSIGN_OR_THROW(outputSchema_, arrow::ImportSchema(&arrowSchema));
}

Expand Down
3 changes: 2 additions & 1 deletion cpp/velox/compute/VeloxBackend.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ class VeloxBackend final : public Backend {
std::make_shared<facebook::velox::substrait::SubstraitParser>();

std::shared_ptr<facebook::velox::substrait::SubstraitVeloxPlanConverter> subVeloxPlanConverter_ =
std::make_shared<facebook::velox::substrait::SubstraitVeloxPlanConverter>(GetDefaultWrappedVeloxMemoryPool());
std::make_shared<facebook::velox::substrait::SubstraitVeloxPlanConverter>(
GetDefaultWrappedVeloxMemoryPool().get());

// Cache for tests/benchmark purpose.
std::shared_ptr<const facebook::velox::core::PlanNode> veloxPlan_;
Expand Down
6 changes: 4 additions & 2 deletions cpp/velox/compute/WholeStageResultIterator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,8 @@ WholeStageResultIteratorFirstStage::WholeStageResultIteratorFirstStage(
}

// Set task parameters.
velox::core::PlanFragment planFragment{planNode, velox::core::ExecutionStrategy::kUngrouped, 1};
std::unordered_set<velox::core::PlanNodeId> emptySet;
velox::core::PlanFragment planFragment{planNode, velox::core::ExecutionStrategy::kUngrouped, 1, emptySet};
std::shared_ptr<velox::core::QueryCtx> queryCtx = createNewVeloxQueryCtx();

task_ = std::make_shared<velox::exec::Task>(
Expand Down Expand Up @@ -328,7 +329,8 @@ WholeStageResultIteratorMiddleStage::WholeStageResultIteratorMiddleStage(
const std::string spillDir,
const std::unordered_map<std::string, std::string>& confMap)
: WholeStageResultIterator(pool, planNode, confMap), streamIds_(streamIds) {
velox::core::PlanFragment planFragment{planNode, velox::core::ExecutionStrategy::kUngrouped, 1};
std::unordered_set<velox::core::PlanNodeId> emptySet;
velox::core::PlanFragment planFragment{planNode, velox::core::ExecutionStrategy::kUngrouped, 1, emptySet};
std::shared_ptr<velox::core::QueryCtx> queryCtx = createNewVeloxQueryCtx();

task_ = std::make_shared<velox::exec::Task>(
Expand Down
2 changes: 1 addition & 1 deletion cpp/velox/compute/WholeStageResultIterator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class WholeStageResultIterator {
}

virtual ~WholeStageResultIterator() {
if (task_->isRunning()) {
if (task_ != nullptr && task_->isRunning()) {
// calling .wait() may take no effect in single thread execution mode
task_->requestCancel().wait();
}
Expand Down
8 changes: 3 additions & 5 deletions cpp/velox/jni/JniWrapper.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,7 @@ JNIEXPORT jboolean JNICALL Java_io_glutenproject_vectorized_ExpressionEvaluatorJ

// A query context used for function validation.
velox::core::QueryCtx queryCtx;

auto pool = gluten::GetDefaultWrappedVeloxMemoryPool();

auto pool = gluten::GetDefaultWrappedVeloxMemoryPool().get();
// An execution context used for function validation.
velox::core::ExecCtx execCtx(pool, &queryCtx);

Expand All @@ -108,12 +106,12 @@ Java_io_glutenproject_spark_sql_execution_datasources_velox_DwrfDatasourceJniWra
jobject obj,
jstring file_path,
jlong c_schema) {
auto pool = gluten::GetDefaultWrappedVeloxMemoryPool();
auto pool = gluten::GetDefaultWrappedVeloxMemoryPool().get();
gluten::DwrfDatasource* dwrfDatasource = nullptr;
if (c_schema == -1) {
// Only inspect the schema and not write
dwrfDatasource = new gluten::DwrfDatasource(JStringToCString(env, file_path), nullptr, pool);
// dwrfDatasource->Init( );
// dwrfDatasource->Init();
} else {
auto schema = gluten::JniGetOrThrow(arrow::ImportSchema(reinterpret_cast<struct ArrowSchema*>(c_schema)));
dwrfDatasource = new gluten::DwrfDatasource(JStringToCString(env, file_path), schema, pool);
Expand Down
2 changes: 1 addition & 1 deletion cpp/velox/memory/VeloxColumnarBatch.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ std::shared_ptr<ArrowSchema> VeloxColumnarBatch::exportArrowSchema() {
std::shared_ptr<ArrowArray> VeloxColumnarBatch::exportArrowArray() {
auto out = std::make_shared<ArrowArray>();
EnsureFlattened();
velox::exportToArrow(flattened_, *out, GetDefaultWrappedVeloxMemoryPool());
velox::exportToArrow(flattened_, *out, GetDefaultWrappedVeloxMemoryPool().get());
return out;
}

Expand Down
Loading