From 43decec47c0e04e0e369590b7699e1b0d7a8243c Mon Sep 17 00:00:00 2001 From: gwang Date: Tue, 30 Jun 2020 14:38:02 -0700 Subject: [PATCH 01/11] init version to use graph instead of model_proto for IsOpSupported --- .../nnapi_builtin/builders/model_builder.cc | 61 ++- .../nnapi_builtin/builders/model_builder.h | 9 +- .../builders/node_attr_helper.cc | 59 +++ .../nnapi_builtin/builders/node_attr_helper.h | 17 + .../nnapi_builtin/builders/op_builder.cc | 406 ++++++++++++++++-- .../nnapi/nnapi_builtin/builders/op_builder.h | 3 + .../nnapi_builtin/nnapi_execution_provider.cc | 5 +- 7 files changed, 516 insertions(+), 44 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 98844f9c739dd..6cec8318ac1bc 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -21,8 +21,9 @@ const float* GetTensorFloatDataA(const ONNX_NAMESPACE::TensorProto& tensor) { : tensor.float_data().data(); } -ModelBuilder::ModelBuilder(ONNX_NAMESPACE::ModelProto& model_proto) - : nnapi_(NnApiImplementation()), model_proto_(model_proto) { +ModelBuilder::ModelBuilder(const ONNX_NAMESPACE::ModelProto& model_proto, + const onnxruntime::GraphViewer& graph_view) + : nnapi_(NnApiImplementation()), model_proto_(model_proto), graph_view_(graph_view) { GetAllInitializers(); op_builders_ = CreateOpBuilders(); } @@ -32,13 +33,21 @@ int32_t ModelBuilder::GetAndroidSdkVer() const { } bool ModelBuilder::IsNodeSupported( - const ONNX_NAMESPACE::NodeProto& node) { - if (auto* opBuilder = GetOpBuilder(node)) { - return opBuilder->IsOpSupported(*this, node); + const onnxruntime::Node& node) { + if (auto* op_builder = GetOpBuilder(node)) { + return op_builder->IsOpSupported(*this, node); } else { return false; } } +// bool ModelBuilder::IsNodeSupported( +// const ONNX_NAMESPACE::NodeProto& node) { +// if (auto* op_builder = GetOpBuilder(node)) { +// return op_builder->IsOpSupported(*this, node); +// } else { +// return false; +// } +// } bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, const ONNX_NAMESPACE::ModelProto& model_proto) { @@ -73,12 +82,31 @@ std::vector> ModelBuilder::GetSupportedNodes() { #endif std::vector supported_node_vec; - for (int i = 0; i < model_proto_.graph().node_size(); i++) { - const auto& node(model_proto_.graph().node(i)); - bool supported = IsNodeSupported(node); - LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node.op_type() + // for (int i = 0; i < model_proto_.graph().node_size(); i++) { + // const auto& node(model_proto_.graph().node(i)); + // bool supported = IsNodeSupported(node); + // LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node.op_type() + // << "] index: [" << i + // << "] name: [" << node.name() + // << "] supported: [" << supported + // << "]"; + // if (supported) { + // supported_node_vec.push_back(i); + // } else { + // if (IsValidSupportedNodesVec(supported_node_vec, model_proto_)) { + // supported_node_vecs.push_back(supported_node_vec); + // supported_node_vec.clear(); + // } + // } + // } + const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); + for (size_t i = 0; i < node_indices.size(); i++) { + const auto* node(graph_view_.GetNode(node_indices[i])); + ORT_ENFORCE(nullptr != node, "node should not be null"); + bool supported = IsNodeSupported(*node); + LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node->OpType() << "] index: [" << i - << "] name: [" << node.name() + << "] name: [" << node->Name() << "] supported: [" << supported << "]"; if (supported) { @@ -156,16 +184,16 @@ void ModelBuilder::GetTargetDevices() { const std::string nnapi_cpu("nnapi-reference"); uint32_t num_devices = 0; THROW_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworks_getDeviceCount(&num_devices), - "Getting list of available devices"); + "Getting count of available devices"); for (uint32_t i = 0; i < num_devices; i++) { ANeuralNetworksDevice* device = nullptr; const char* device_name = nullptr; THROW_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworks_getDevice(i, &device), - "Getting list of available devices"); + "Getting " + std::to_string(i) + "th device"); THROW_ON_ERROR_WITH_NOTE(nnapi_->ANeuralNetworksDevice_getName(device, &device_name), - "Getting list of available devices"); + "Getting " + std::to_string(i) + "th device's name"); bool device_is_cpu = nnapi_cpu == device_name; if ((target_device_option_ == TargetDeviceOption::CPU_DISABLED && !device_is_cpu) || @@ -516,6 +544,13 @@ IOpBuilder* ModelBuilder::GetOpBuilder(const ONNX_NAMESPACE::NodeProto& node) { return op_builders_[node.op_type()].get(); } +IOpBuilder* ModelBuilder::GetOpBuilder(const onnxruntime::Node& node) { + if (!Contains(op_builders_, node.OpType())) + return nullptr; + + return op_builders_[node.OpType()].get(); +} + std::string ModelBuilder::GetUniqueName(const std::string& base_name) { std::string unique_name; do { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index d9ca4a1b691a0..579c78f9b2d3a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -5,6 +5,7 @@ #include #include +#include #include "core/providers/nnapi/nnapi_builtin/model.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" #include "op_builder.h" @@ -26,7 +27,7 @@ class ModelBuilder { CPU_ONLY, // use CPU only }; - ModelBuilder(ONNX_NAMESPACE::ModelProto& model_proto); + ModelBuilder(const ONNX_NAMESPACE::ModelProto& model_proto, const onnxruntime::GraphViewer& graph_view); ~ModelBuilder() = default; std::vector> GetSupportedNodes(); @@ -94,6 +95,7 @@ class ModelBuilder { GetInitializerTensors() const { return initializers_; } const ONNX_NAMESPACE::ModelProto& GetOnnxModel() const { return model_proto_; } + const onnxruntime::Graph& GetOnnxGraph() const { return graph_view_.GetGraph(); } void RegisterNHWCOperand(const std::string& name); bool IsOperandNHWC(const std::string& name); @@ -109,7 +111,8 @@ class ModelBuilder { private: const NnApi* nnapi_{nullptr}; - ONNX_NAMESPACE::ModelProto& model_proto_; + const ONNX_NAMESPACE::ModelProto& model_proto_; + const onnxruntime::GraphViewer& graph_view_; std::unique_ptr nnapi_model_; uint32_t name_token_{0}; @@ -150,6 +153,7 @@ class ModelBuilder { uint32_t next_index_ = 0; bool IsNodeSupported(const ONNX_NAMESPACE::NodeProto& node); + bool IsNodeSupported(const onnxruntime::Node& node); // Convert the onnx model to ANeuralNetworksModel void Prepare(); @@ -172,6 +176,7 @@ class ModelBuilder { bool is_nhwc); IOpBuilder* GetOpBuilder(const ONNX_NAMESPACE::NodeProto& node); + IOpBuilder* GetOpBuilder(const onnxruntime::Node& node); }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc index 7fee45cec4caf..799686c38a5dd 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc @@ -7,6 +7,7 @@ #include #include "core/common/safeint.h" +#include "helper.h" #include "node_attr_helper.h" using std::string; @@ -102,3 +103,61 @@ bool NodeAttrHelper::HasAttr(const std::string& key) { return false; } + +GraphNodeAttrHelper::GraphNodeAttrHelper(const onnxruntime::Node& node) + : node_attributes_(node.GetAttributes()) {} + +float GraphNodeAttrHelper::Get(const std::string& key, float def_val) const { + if (HasAttr(key)) + return node_attributes_.at(key).f(); + + return def_val; +} + +int32_t GraphNodeAttrHelper::Get(const std::string& key, int32_t def_val) const { + if (HasAttr(key)) + return SafeInt(node_attributes_.at(key).i()); + + return def_val; +} + +string GraphNodeAttrHelper::Get(const std::string& key, const string& def_val) const { + if (HasAttr(key)) + return node_attributes_.at(key).s(); + + return def_val; +} + +vector GraphNodeAttrHelper::Get(const std::string& key, const vector& def_val) const { + if (HasAttr(key)) { + const auto& attr(node_attributes_.at(key)); + std::vector v; + v.reserve(static_cast(attr.ints_size())); + for (int j = 0; j < attr.ints_size(); j++) { + int64_t val = attr.ints(j); + v.push_back(SafeInt(val)); + } + return v; + } + + return def_val; +} + +vector GraphNodeAttrHelper::Get(const std::string& key, const vector& def_val) const { + if (HasAttr(key)) { + const auto& attr(node_attributes_.at(key)); + std::vector v; + v.reserve(static_cast(attr.ints_size())); + for (int j = 0; j < attr.ints_size(); j++) { + v.push_back(attr.floats(j)); + } + + return v; + } + + return def_val; +} + +bool GraphNodeAttrHelper::HasAttr(const std::string& key) const { + return Contains(node_attributes_, key); +} diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h index 80594bc45ac9e..7e18b0d87764c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include @@ -25,3 +26,19 @@ class NodeAttrHelper { private: const ONNX_NAMESPACE::NodeProto& node_; }; + +class GraphNodeAttrHelper { + public: + GraphNodeAttrHelper(const onnxruntime::Node& proto); + + float Get(const std::string& key, float def_val) const; + int32_t Get(const std::string& key, int32_t def_val) const; + std::vector Get(const std::string& key, const std::vector& def_val) const; + std::vector Get(const std::string& key, const std::vector& def_val) const; + std::string Get(const std::string& key, const std::string& def_val) const; + + bool HasAttr(const std::string& key) const; + + private: + const onnxruntime::NodeAttributes& node_attributes_; +}; \ No newline at end of file diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 3b69ee44a6566..7316b73091d00 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -150,6 +150,30 @@ int GetType(const ONNX_NAMESPACE::ModelProto& model_proto, return invalid_type; } +bool GetType(const onnxruntime::Node& node, int32_t& type) { + type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; + auto* type_proto = node.InputDefs()[0]->TypeAsProto(); + if (!type_proto || !type_proto->tensor_type().has_elem_type()) { + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] has no input type"; + return false; + } + + type = type_proto->tensor_type().elem_type(); + return true; +} + +Shaper::Shape GetShape(const ONNX_NAMESPACE::TensorShapeProto* shape_proto) { + Shaper::Shape shape; + if (!shape_proto) + return shape; + + for (const auto& dim : shape_proto->dim()) + // NNAPI uses 0 for dynamic dimension, which is the default value for dim.dim_value() + shape.push_back(SafeInt(dim.dim_value())); + + return shape; +} + Shaper::Shape GetShape(const ONNX_NAMESPACE::ModelProto& model_proto, const std::string& name) { Shaper::Shape empty_shape; @@ -318,6 +342,9 @@ class BaseOpBuilder : public IOpBuilder { bool IsOpSupported(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override final; + bool IsOpSupported(ModelBuilder& model_builder, + const onnxruntime::Node& node) override final; + void AddToModelBuilder(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override final; @@ -325,12 +352,20 @@ class BaseOpBuilder : public IOpBuilder { virtual bool IsOpSupportedImpl( ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); + virtual bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node); + virtual int32_t GetMinSupportedSdkVer( ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const { return 27; } - virtual bool HasSupportedInputs( - ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); + virtual int32_t GetMinSupportedSdkVer( + ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) const { return 27; } + + virtual bool HasSupportedInputs(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); + + virtual bool HasSupportedInputs(const onnxruntime::Node& node); virtual void AddToModelBuilderImpl( ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); @@ -353,7 +388,26 @@ bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, return false; return IsOpSupportedImpl(model_builder, node); -} // namespace nnapi +} + +bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, + const onnxruntime::Node& node) { +#ifdef __ANDROID__ + int32_t android_sdk_ver = model_builder.GetAndroidSdkVer(); + int32_t required_sdk_ver = GetMinSupportedSdkVer(model_builder, node); + if (required_sdk_ver > android_sdk_ver) { + LOGS_DEFAULT(VERBOSE) << "Current Android API level [" << android_sdk_ver + << "], Operator [" << node.OpType() + << "] is only supported on API >" << required_sdk_ver; + return false; + } +#endif + + if (!HasSupportedInputs(node)) + return false; + + return IsOpSupportedImpl(model_builder, node); +} bool BaseOpBuilder::HasSupportedInputs( ModelBuilder& model_builder, @@ -371,11 +425,33 @@ bool BaseOpBuilder::HasSupportedInputs( return true; } +bool BaseOpBuilder::HasSupportedInputs(const onnxruntime::Node& node) { + // We only check the type of input 0 by default + // specific op builder can override this + int32_t input_type; + if (!GetType(node, input_type)) + return false; + + if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + << "] Input type: [" << input_type + << "] is not supported for now"; + return false; + } + + return true; +} + bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) { return true; } +bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) { + return true; +} + void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { ORT_ENFORCE(IsOpSupported(model_builder, node), @@ -497,9 +573,11 @@ void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class TransposeOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; + + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { @@ -510,6 +588,18 @@ class TransposeOpBuilder : public BaseOpBuilder { const ONNX_NAMESPACE::NodeProto& node) override; }; +bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) { + const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + if (input_size > 4 || input_size == 0) { + LOGS_DEFAULT(VERBOSE) << "Transpose only supports 1-4d shape, input is " + << input_size << "d shape"; + return false; + } + + return true; +} + bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); @@ -566,9 +656,12 @@ class ReshapeOpBuilder : public BaseOpBuilder { const ONNX_NAMESPACE::NodeProto& node) override; private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; + + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; }; @@ -578,6 +671,37 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, model_builder.AddInitializerToSkip(node.input(1)); } +bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + const auto& initializers(model_builder.GetInitializerTensors()); + const auto& perm_name = node.InputDefs()[1]->Name(); + if (!Contains(initializers, perm_name)) { + LOGS_DEFAULT(VERBOSE) << "New shape of reshape must be known"; + return false; + } + + const auto input_shape = GetShape(node.InputDefs()[0]->Shape()); + if (input_shape.size() > 4 || input_shape.empty()) { + LOGS_DEFAULT(VERBOSE) << "Reshape only supports up to 1-4d shape, input is " + << input_shape.size() << "d shape"; + return false; + } + + const auto& shape_tensor = initializers.at(perm_name); + const int64_t* rawShape = GetTensorInt64Data(shape_tensor); + const auto size = SafeInt(shape_tensor.dims()[0]); + + for (uint32_t i = 0; i < size; i++) { + // NNAPI reshape does not support 0 as dimension + if (rawShape[i] == 0 && i < input_shape.size() && input_shape[i] == 0) { + LOGS_DEFAULT(VERBOSE) << "Reshape doesn't suppport 0 reshape dimension on a dynamic dimension"; + return false; + } + } + + return true; +} + bool ReshapeOpBuilder::IsOpSupportedImpl( ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { @@ -669,6 +793,10 @@ class BatchNormalizationOpBuilder : public BaseOpBuilder { private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; }; @@ -682,6 +810,39 @@ void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_buil model_builder.AddInitializerToSkip(node.input(4)); //var } +bool BatchNormalizationOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + if (node.OutputDefs().size() != 1) { + LOGS_DEFAULT(VERBOSE) << "Your onnx model may be in training mode, please export " + "it in test mode."; + return false; + } + + const auto& initializers(model_builder.GetInitializerTensors()); + const auto& scale_name = node.InputDefs()[1]->Name(); + const auto& b_name = node.InputDefs()[2]->Name(); + const auto& mean_name = node.InputDefs()[3]->Name(); + const auto& var_name = node.InputDefs()[4]->Name(); + if (!Contains(initializers, scale_name)) { + LOGS_DEFAULT(VERBOSE) << "Scale of BN must be known"; + return false; + } + if (!Contains(initializers, b_name)) { + LOGS_DEFAULT(VERBOSE) << "B of BN must be known"; + return false; + } + if (!Contains(initializers, mean_name)) { + LOGS_DEFAULT(VERBOSE) << "Mean of BN must be known"; + return false; + } + if (!Contains(initializers, var_name)) { + LOGS_DEFAULT(VERBOSE) << "Var of BN must be known"; + return false; + } + + return true; +} + bool BatchNormalizationOpBuilder::IsOpSupportedImpl( ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { @@ -793,9 +954,11 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil class PoolOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; + + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { @@ -806,6 +969,62 @@ class PoolOpBuilder : public BaseOpBuilder { const ONNX_NAMESPACE::NodeProto& node) override; }; +bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) { + const auto& op = node.OpType(); + if (op == "AveragePool" || op == "MaxPool") { + GraphNodeAttrHelper helper(node); + + const auto count_include_pad = helper.Get("count_include_pad", 0); + if (count_include_pad == 1) { + LOGS_DEFAULT(VERBOSE) << "count_include_pad == 1 is not supported"; + return false; + } + + const auto storage_order = helper.Get("storage_order", 0); + if (storage_order == 1) { + LOGS_DEFAULT(VERBOSE) << "storage_order == 1 is not supported"; + return false; + } + + if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { + LOGS_DEFAULT(VERBOSE) << "auto_pad is not supported"; + return false; + } + + if (helper.Get("kernel_shape", std::vector{1, 1}).size() != 2) { + LOGS_DEFAULT(VERBOSE) << "Only pooling 2d is supported"; + return false; + } + + if (helper.Get("ceil_mode", 0) == 1) { + LOGS_DEFAULT(VERBOSE) << "ceil_mode == 1 is not supported for pooling"; + return false; + } + + if (helper.Get("dilations", std::vector{1, 1}) != + std::vector{1, 1}) { + LOGS_DEFAULT(VERBOSE) << "Dilations of pooling is not supported"; + return false; + } + + if (node.OutputDefs().size() != 1) { + LOGS_DEFAULT(VERBOSE) << "Argmax in maxpooling is not supported"; + return false; + } + } else if (op == "GlobalAveragePool" || op == "GlobalMaxPool") { + const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + if (input_size != 4) { + LOGS_DEFAULT(VERBOSE) + << "GlobalAveragePool/GlobalMaxPool Only rank-4 tensor is supported in " + << node.InputDefs()[0]->Name() << ", actual dim count " << input_size; + return false; + } + } + + return true; +} + bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { const auto& op = node.op_type(); @@ -946,6 +1165,10 @@ class ConvOpBuilder : public BaseOpBuilder { private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; }; @@ -956,9 +1179,36 @@ void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, model_builder.AddInitializerToSkip(node.input(1)); } -bool ConvOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + GraphNodeAttrHelper helper(node); + if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { + LOGS_DEFAULT(VERBOSE) << "SAME_LOWER auto_pad is not supported"; + return false; + } + + const auto group = helper.Get("group", 1); + const auto weight_name = node.InputDefs()[1]->Name(); + if (Contains(model_builder.GetInitializerTensors(), weight_name)) { + const auto& tensor = model_builder.GetInitializerTensors().at(weight_name); + if (tensor.dims().size() != 4) { + LOGS_DEFAULT(VERBOSE) << "Only conv 2d is supported."; + return false; + } + if (group != 1 && tensor.dims()[1] != 1) { + LOGS_DEFAULT(VERBOSE) << "group != 1 is not supported"; + return false; + } + } else { + LOGS_DEFAULT(VERBOSE) << "The weight of convolution must be known"; + return false; + } + + return true; +} + +bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) { NodeAttrHelper helper(node); if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { LOGS_DEFAULT(VERBOSE) << "SAME_LOWER auto_pad is not supported"; @@ -1112,9 +1362,11 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class CastOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; + + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { @@ -1125,9 +1377,21 @@ class CastOpBuilder : public BaseOpBuilder { const ONNX_NAMESPACE::NodeProto& node) override; }; -bool CastOpBuilder::IsOpSupportedImpl( - ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) { +bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) { + GraphNodeAttrHelper helper(node); + auto to = helper.Get("to", 0); + if (to != ONNX_NAMESPACE::TensorProto::FLOAT && + to != ONNX_NAMESPACE::TensorProto::INT32) { + LOGS_DEFAULT(VERBOSE) << "[Cast] Only support cast to int32 or float"; + return false; + } + + return true; +} + +bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, + const ONNX_NAMESPACE::NodeProto& node) { NodeAttrHelper helper(node); auto to = helper.Get("to", 0); if (to != ONNX_NAMESPACE::TensorProto::FLOAT && @@ -1177,9 +1441,11 @@ void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class SoftMaxOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; + + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { @@ -1190,9 +1456,19 @@ class SoftMaxOpBuilder : public BaseOpBuilder { const ONNX_NAMESPACE::NodeProto& node) override; }; -bool SoftMaxOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { +bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) { + const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + if (input_size != 2 && input_size != 4) { + LOGS_DEFAULT(VERBOSE) << "SoftMax only support 2d/4d shape, input is " + << input_size << "d shape"; + return false; + } + return true; +} + +bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) { const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); if (input_size != 2 && input_size != 4) { LOGS_DEFAULT(VERBOSE) << "SoftMax only support 2d/4d shape, input is " @@ -1276,10 +1552,71 @@ class GemmOpBuilder : public BaseOpBuilder { bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; }; +bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + const auto& op = node.OpType(); + const auto& initializers(model_builder.GetInitializerTensors()); + + if (GetShape(node.InputDefs()[0]->Shape()).size() != 2) { + LOGS_DEFAULT(VERBOSE) << "A must be 2D"; + return false; + } + + if (GetShape(node.InputDefs()[1]->Shape()).size() != 2) { + LOGS_DEFAULT(VERBOSE) << "B must be 2D"; + return false; + } + + if (op == "MatMul") { // Only support A*B B is an initializer + if (!Contains(initializers, node.InputDefs()[1]->Name())) { + LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known"; + return false; + } + } else if (op == "Gemm") { + // Only support + // 1. A*B'+C + // 2. A*B+C and B is an initializer + GraphNodeAttrHelper helper(node); + const auto transA = helper.Get("transA", 0); + const auto transB = helper.Get("transB", 0); + const auto alpha = helper.Get("alpha", 1.0f); + const auto beta = helper.Get("beta", 1.0f); + + if (!(transA == 0 && alpha == 1.f && beta == 1.f)) { + LOGS_DEFAULT(VERBOSE) << "Only transA == 0, alpha == 1.0 " + << "and beta == 1.0 is supported."; + return false; + } + + if (transB == 0 && !Contains(initializers, node.InputDefs()[1]->Name())) { + LOGS_DEFAULT(VERBOSE) << "B of Gemm must be known if transB != 1"; + return false; + } + + if (node.InputDefs().size() == 3) { + const auto b_shape = GetShape(node.InputDefs()[1]->Shape()); + const auto c_shape = GetShape(node.InputDefs()[2]->Shape()); + if (c_shape.size() != 1 || + c_shape[0] != (transB == 0 ? b_shape[1] : b_shape[0])) { + LOGS_DEFAULT(VERBOSE) << "C of Gemm must be a vector of b_shape[0]" + << " b_shape: " << Shape2String(b_shape) + << " c_shape: " << Shape2String(c_shape); + + return false; + } + } + } + + return true; +} + bool GemmOpBuilder::IsOpSupportedImpl( ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { @@ -1484,10 +1821,25 @@ class ConcatOpBuilder : public BaseOpBuilder { bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; }; +bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) { + auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + if (input_size > 4 || input_size == 0) { + LOGS_DEFAULT(VERBOSE) << "Concat only supports up to 1-4d shape, input is " + << input_size << "d shape"; + return false; + } + + return true; +} + bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { if (GetShape(model_builder.GetOnnxModel(), node.input(0)).size() > 4) { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index d0901ee1c150d..e338f63079248 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -16,6 +16,9 @@ class IOpBuilder { virtual bool IsOpSupported(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) = 0; + virtual bool IsOpSupported(ModelBuilder& model_builder, + const onnxruntime::Node& node) = 0; + // Check if the initializers of this operator need preprocess // which will not be copied virtual void AddInitializersToSkip(ModelBuilder& model_builder, diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 91201cb51779b..20ebee5b690e6 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -87,7 +87,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); - nnapi::ModelBuilder builder(model_proto); + nnapi::ModelBuilder builder(model_proto, graph_view); const auto supported_nodes_vector = builder.GetSupportedNodes(); // Find inputs, initializers and outputs for each supported subgraph @@ -227,7 +227,8 @@ common::Status NnapiExecutionProvider::Compile(const std::vector nnapi_model = builder.Compile(); From f43248f6b413c9ab25fcb3608479233490579a65 Mon Sep 17 00:00:00 2001 From: gwang Date: Tue, 30 Jun 2020 17:40:03 -0700 Subject: [PATCH 02/11] move add to modelbuilder to use graph node --- .../nnapi_builtin/builders/model_builder.cc | 20 +- .../nnapi_builtin/builders/op_builder.cc | 809 +++++++++++++++++- .../nnapi/nnapi_builtin/builders/op_builder.h | 4 + 3 files changed, 809 insertions(+), 24 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 6cec8318ac1bc..ed864d3d5b1f2 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -427,14 +427,26 @@ uint32_t ModelBuilder::AddOperandFromPersistMemoryBuffer( } void ModelBuilder::AddOperations() { - for (const auto& node : model_proto_.graph().node()) { - if (auto* opBuilder = GetOpBuilder(node)) { - opBuilder->AddToModelBuilder(*this, node); + const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); + for (size_t i = 0; i < node_indices.size(); i++) { + const auto* node(graph_view_.GetNode(node_indices[i])); + ORT_ENFORCE(nullptr != node, "node should not be null"); + if (auto* opBuilder = GetOpBuilder(*node)) { + opBuilder->AddToModelBuilder(*this, *node); } else { throw std::invalid_argument( - "Node not supported" + node.name()); + "Node not supported" + node->Name()); } } + + // for (const auto& node : model_proto_.graph().node()) { + // if (auto* opBuilder = GetOpBuilder(node)) { + // opBuilder->AddToModelBuilder(*this, node); + // } else { + // throw std::invalid_argument( + // "Node not supported" + node.name()); + // } + // } } void ModelBuilder::AddOperation(int op, const std::vector& input_indices, diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 7316b73091d00..ee0be94d98746 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -348,6 +348,9 @@ class BaseOpBuilder : public IOpBuilder { void AddToModelBuilder(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override final; + void AddToModelBuilder(ModelBuilder& model_builder, + const onnxruntime::Node& node) override final; + protected: virtual bool IsOpSupportedImpl( ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); @@ -369,6 +372,9 @@ class BaseOpBuilder : public IOpBuilder { virtual void AddToModelBuilderImpl( ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); + + virtual void AddToModelBuilderImpl( + ModelBuilder& model_builder, const onnxruntime::Node& node); }; bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, @@ -452,6 +458,16 @@ bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } +void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + ORT_ENFORCE(IsOpSupported(model_builder, node), + "Unsupported operator " + node.OpType()); + + AddToModelBuilderImpl(model_builder, node); + LOGS_DEFAULT(VERBOSE) << "Operator name: [" << node.Name() + << "] type: [" << node.OpType() << "] was added"; +} + void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { ORT_ENFORCE(IsOpSupported(model_builder, node), @@ -467,6 +483,11 @@ void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, ORT_NOT_IMPLEMENTED("Unsupported operator " + node.op_type()); } +void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) { + ORT_NOT_IMPLEMENTED("Unsupported operator " + node.OpType()); +} + #pragma endregion op_base #pragma region op_binary @@ -476,11 +497,27 @@ class BinaryOpBuilder : public BaseOpBuilder { int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) const override; + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, + const onnxruntime::Node& node) const override; + private: void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; +int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) const { + const auto& op(node.OpType()); + if (op == "Sub" || op == "Div") { + return 28; + } + + return 27; +} + int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& node) const { const auto& op(node.op_type()); @@ -491,6 +528,51 @@ int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */ return 27; } +void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + const auto& op(node.OpType()); + int32_t op_code; + if (op == "Add") + op_code = ANEURALNETWORKS_ADD; + else if (op == "Sub") + op_code = ANEURALNETWORKS_SUB; + else if (op == "Mul") + op_code = ANEURALNETWORKS_MUL; + else if (op == "Div") + op_code = ANEURALNETWORKS_DIV; + else { + ORT_THROW("UnaryOpBuilder, unknown op: " + op); + } + std::string input1 = node.InputDefs()[0]->Name(); + std::string input2 = node.InputDefs()[1]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + + bool input1_is_nhwc = model_builder.IsOperandNHWC(input1); + bool input2_is_nhwc = model_builder.IsOperandNHWC(input2); + bool output_is_nhwc = false; + + if (input1_is_nhwc == input2_is_nhwc) { + output_is_nhwc = input1_is_nhwc; + } else if (input1_is_nhwc) { + // need transpsoe input1 back to nchw + const auto& nhwc_input = node.InputDefs()[0]->Name(); + if (!model_builder.GetNCHWOperand(nhwc_input, input1)) { + input1 = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); + TransposeNHWCToNCHW(model_builder, nhwc_input, input1); + } + } else { // input2_is_nhwc + // need transpsoe input2 back to nchw + const auto& nhwc_input = node.InputDefs()[1]->Name(); + if (!model_builder.GetNCHWOperand(nhwc_input, input2)) { + input2 = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); + TransposeNHWCToNCHW(model_builder, nhwc_input, input2); + } + } + + int32_t fuse_code = model_builder.FindActivation(output); + AddBinaryOperator(op_code, model_builder, input1, input2, fuse_code, output, output_is_nhwc); +} + void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { const auto& op(node.op_type()); @@ -543,8 +625,33 @@ class ReluOpBuilder : public BaseOpBuilder { private: void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; +void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + bool output_is_nhwc = model_builder.IsOperandNHWC(input); + shaper.Identity(input, output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + + // skip this relu if it is some op's fuse output + if (Contains(model_builder.GetFusedActivations(), node.Name())) { + model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); + } else { + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + model_builder.AddOperation(ANEURALNETWORKS_RELU, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); + } +} + void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -579,6 +686,11 @@ class TransposeOpBuilder : public BaseOpBuilder { bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) const override { + return 28; + } + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { return 28; @@ -586,6 +698,9 @@ class TransposeOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, @@ -612,6 +727,40 @@ bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } +void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + + auto input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + GraphNodeAttrHelper helper(node); + vector perm = helper.Get("perm", vector()); + auto input_dims = shaper[input].size(); + if (perm.empty()) { + for (int32_t i = input_dims - 1; i >= 0; i--) + perm.push_back(i); + } else { + ORT_ENFORCE(perm.size() == input_dims, "Perm and input should have same dimension"); + } + + if (model_builder.IsOperandNHWC(input)) { + ORT_ENFORCE(input_dims == 4, "Only 4D shape can be nhwc"); + + // we are using nhwc here, but the axis is in nchw, need to transpose axis from nchw to nhwc + const int32_t axis_nchw_to_nhwc[4]{0, 3, 1, 2}; + for (size_t i = 0; i < perm.size(); i++) + perm[i] = axis_nchw_to_nhwc[perm[i]]; + } + + std::string perm_name = model_builder.GetUniqueName(node.Name() + input + "perm"); + + // It is possible this onnx transpose operator can be nchw->nhwc, but so far I don't see + // any scenario will do this since onnx is nchw only, assume the output is always not nhwc + // even it is, there will be extra transpose in the onnx model to convert it back to nchw + // before conv/pool/... operators + AddTransposeOperator(model_builder, input, perm_name, perm, output, false /* is_nhwc */); +} + void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -664,6 +813,9 @@ class ReshapeOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, @@ -735,6 +887,52 @@ bool ReshapeOpBuilder::IsOpSupportedImpl( return true; } +void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + const auto& initializers(model_builder.GetInitializerTensors()); + + auto input = node.InputDefs()[0]->Name(); + + if (model_builder.IsOperandNHWC(input)) { + // We want to transpose nhwc operand back to nchw before reshape + const auto& nhwc_input = node.InputDefs()[0]->Name(); + if (!model_builder.GetNCHWOperand(nhwc_input, input)) { + input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); + TransposeNHWCToNCHW(model_builder, nhwc_input, input); + } + } + + const auto& output = node.OutputDefs()[0]->Name(); + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); // input + + const auto& shape_tensor = initializers.at(node.InputDefs()[1]->Name()); + const int64_t* rawShape = GetTensorInt64Data(shape_tensor); + const auto size = SafeInt(shape_tensor.dims()[0]); + + ModelBuilder::Shape input_shape = shaper[input]; + std::vector shape(size); + for (uint32_t i = 0; i < size; i++) { + int32_t dim = SafeInt(rawShape[i]); + // NNAPI reshape does not support 0 as dimension + shape[i] = dim == 0 ? input_shape[i] : dim; + } + + ModelBuilder::Shape shape_dimen = {size}; + std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape"); + OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen); + uint32_t shape_idx = model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type); + input_indices.push_back(shape_idx); + + shaper.Reshape(input, shape, output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices, + {output}, {output_operand_type}, {false}); +} + void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -799,6 +997,9 @@ class BatchNormalizationOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, @@ -877,6 +1078,77 @@ bool BatchNormalizationOpBuilder::IsOpSupportedImpl( return true; } +void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_types(model_builder.GetOperandTypes()); + const auto& initializers(model_builder.GetInitializerTensors()); + GraphNodeAttrHelper helper(node); + + // For reshape we are not really doing anything but + // register a new operand with new shape + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + + const auto& scale_tensor = initializers.at(node.InputDefs()[1]->Name()); + const auto& bias_tensor = initializers.at(node.InputDefs()[2]->Name()); + const auto& mean_tensor = initializers.at(node.InputDefs()[3]->Name()); + const auto& var_tensor = initializers.at(node.InputDefs()[4]->Name()); + const auto eps = helper.Get("epsilon", 1e-5f); + + const auto size = SafeInt(scale_tensor.dims()[0]); + vector a, b; + a.reserve(size); + b.reserve(size); + + const float* scale_data = GetTensorFloatData(scale_tensor); + const float* bias_data = GetTensorFloatData(bias_tensor); + const float* mean_data = GetTensorFloatData(mean_tensor); + const float* var_data = GetTensorFloatData(var_tensor); + + for (int64_t i = 0; i < size; i++) { + a.push_back(scale_data[i] / sqrt(var_data[i] + eps)); + b.push_back((scale_data[i] * -mean_data[i]) / sqrt(var_data[i] + eps) + + bias_data[i]); + } + + const auto tensor_a_name = model_builder.GetUniqueName(node.Name() + input + "_imm_a"); + const auto tensor_b_name = model_builder.GetUniqueName(node.Name() + input + "_imm_b"); + const auto tensor_imm_product_name = model_builder.GetUniqueName(node.Name() + input + "_imm_mul"); + ModelBuilder::Shape tensor_a_dimen; + + bool input_is_nhwc = model_builder.IsOperandNHWC(input); + bool output_is_nhwc = input_is_nhwc; + if (input_is_nhwc) + tensor_a_dimen = {size}; + else // input is nchw + tensor_a_dimen = {size, 1, 1}; // {C, H, W} + + shaper.AddShape(tensor_a_name, tensor_a_dimen); + shaper.AddShape(tensor_b_name, tensor_a_dimen); + const OperandType operandType_a(operand_types.at(input).type, tensor_a_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(tensor_a_name, a.data(), operandType_a); + const OperandType operandType_b(operand_types.at(input).type, tensor_a_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), operandType_b); + + // Mul + AddBinaryOperator(ANEURALNETWORKS_MUL, + model_builder, + input, tensor_a_name, + ANEURALNETWORKS_FUSED_NONE, + tensor_imm_product_name, + output_is_nhwc); + + // Add + int32_t fuse_code = model_builder.FindActivation(output); + AddBinaryOperator(ANEURALNETWORKS_ADD, + model_builder, + tensor_imm_product_name, tensor_b_name, + fuse_code, + output, + output_is_nhwc); +} + void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -960,6 +1232,11 @@ class PoolOpBuilder : public BaseOpBuilder { bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) const override { + return 28; + } + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { return 29; @@ -967,6 +1244,9 @@ class PoolOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, @@ -1082,14 +1362,14 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, } void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { + const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - NodeAttrHelper helper(node); + GraphNodeAttrHelper helper(node); - auto input = node.input(0); + auto input = node.InputDefs()[0]->Name(); bool use_nchw = model_builder.UseNCHW(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = false; @@ -1098,7 +1378,7 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } else { output_is_nhwc = true; if (!input_is_nhwc) { - const auto& nchw_input = node.input(0); + const auto& nchw_input = node.InputDefs()[0]->Name(); if (!model_builder.GetNHWCOperand(nchw_input, input)) { input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); TransposeNCHWToNHWC(model_builder, nchw_input, input); @@ -1106,8 +1386,8 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } } - const auto& output = node.output(0); - const auto& op = node.op_type(); + const auto& output = node.OutputDefs()[0]->Name(); + const auto& op = node.OpType(); int32_t op_type; if (op == "AveragePool" || op == "GlobalAveragePool") @@ -1153,24 +1433,99 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -#pragma endregion op_pool - -#pragma region op_conv - -class ConvOpBuilder : public BaseOpBuilder { - public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; +void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); - private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + NodeAttrHelper helper(node); + + auto input = node.input(0); + bool use_nchw = model_builder.UseNCHW(); + bool input_is_nhwc = model_builder.IsOperandNHWC(input); + bool output_is_nhwc = false; + if (use_nchw) { + ORT_ENFORCE(!input_is_nhwc, "model_builder.UseNCHW() but input is NHWC"); + } else { + output_is_nhwc = true; + if (!input_is_nhwc) { + const auto& nchw_input = node.input(0); + if (!model_builder.GetNHWCOperand(nchw_input, input)) { + input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); + TransposeNCHWToNHWC(model_builder, nchw_input, input); + } + } + } + + const auto& output = node.output(0); + const auto& op = node.op_type(); + + int32_t op_type; + if (op == "AveragePool" || op == "GlobalAveragePool") + op_type = ANEURALNETWORKS_AVERAGE_POOL_2D; + else // (op == "MaxPool" || op == "GlobalMaxPool") + op_type = ANEURALNETWORKS_MAX_POOL_2D; + + vector onnx_pads, onnx_strides, kernel_shape; + if (op == "AveragePool" || op == "MaxPool") { + kernel_shape = helper.Get("kernel_shape", vector{0, 0}); + onnx_strides = helper.Get("strides", vector{1, 1}); + onnx_pads = helper.Get("pads", vector{0, 0, 0, 0}); + } else { // (op == "GlobalAveragePool" || op == "GlobalMaxPool") + onnx_strides = vector{1, 1}; + onnx_pads = vector{0, 0, 0, 0}; + if (model_builder.UseNCHW()) + kernel_shape = vector{static_cast(shaper[input][2]), + static_cast(shaper[input][3])}; + else + kernel_shape = vector{static_cast(shaper[input][1]), + static_cast(shaper[input][2])}; + } + + int32_t fuse_code = model_builder.FindActivation(output); + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[1])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[3])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[0])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[2])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[1])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[0])); + input_indices.push_back(model_builder.AddOperandFromScalar(kernel_shape[1])); + input_indices.push_back(model_builder.AddOperandFromScalar(kernel_shape[0])); + input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); + input_indices.push_back(model_builder.AddOperandFromScalar(use_nchw)); + + shaper.Pool(input, + onnx_pads, onnx_strides, kernel_shape, + use_nchw, + output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); +} + +#pragma endregion op_pool + +#pragma region op_conv + +class ConvOpBuilder : public BaseOpBuilder { + public: + void AddInitializersToSkip(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; + + private: + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, @@ -1235,6 +1590,127 @@ bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } +void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + const auto& initializers(model_builder.GetInitializerTensors()); + GraphNodeAttrHelper helper(node); + + // onnx strides are in the order height, width + // while nnapi strides are in the order width, height + const auto onnx_strides = helper.Get("strides", vector{1, 1}); + + // onnx pads are in the order top, left, bottom, right + // while nnapi pads is in the order left, right, top, bottom + const auto onnx_pads = helper.Get("pads", vector{0, 0, 0, 0}); + + // onnx dilations is in the order height, width + // while nnapi dilations are in the order width, height + const auto onnx_dilations = helper.Get("dilations", vector{1, 1}); + const auto group = helper.Get("group", 1); + + auto input = node.InputDefs()[0]->Name(); + bool use_nchw = model_builder.UseNCHW(); + bool input_is_nhwc = model_builder.IsOperandNHWC(input); + bool output_is_nhwc = false; + if (use_nchw) { + ORT_ENFORCE(!input_is_nhwc, "model_builder.UseNCHW() but input is NHWC"); + } else { + output_is_nhwc = true; + if (!input_is_nhwc) { + const auto& nchw_input = node.InputDefs()[0]->Name(); + if (!model_builder.GetNHWCOperand(nchw_input, input)) { + input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); + TransposeNCHWToNHWC(model_builder, nchw_input, input); + } + } + } + + const auto& weight = node.InputDefs()[1]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + + bool conv2d = (group == 1); + const auto& weight_tensor = initializers.at(weight); + bool depthwise_conv2d = (weight_tensor.dims()[1] == 1); + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + + if (conv2d) { + input_indices.push_back(AddInitializerInNewLayout( + model_builder, weight, L_0231)); + } else { // depthwise_conv2d + input_indices.push_back(AddInitializerInNewLayout( + model_builder, weight, L_1230)); + } + + bool hasBias = (node.InputDefs().size() >= 3); + std::string bias = hasBias ? node.InputDefs()[2]->Name() : weight + "_bias"; + + uint32_t bias_idx_val; + if (hasBias) { + bias_idx_val = operand_indices.at(bias); + } else { + const auto weight_dimen = shaper[weight]; + ModelBuilder::Shape bias_dimen; + if (conv2d) + bias_dimen = {weight_dimen[0]}; + else + bias_dimen = {weight_dimen[3]}; + + const auto& weight_type = operand_types.at(weight).type; + if (weight_type == Type::TENSOR_FLOAT32) { + vector buffer(bias_dimen[0]); + for (uint32_t i = 0; i < buffer.size(); i++) { + buffer[i] = 0.f; + } + OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); + bias_idx_val = model_builder.AddOperandFromPersistMemoryBuffer( + bias, buffer.data(), operandType); + } else { + ORT_THROW("Unknown weight type " + TypeToStr(weight_type)); + } + } + + input_indices.push_back(bias_idx_val); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[1])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[3])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[0])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[2])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[1])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[0])); + if (!conv2d && depthwise_conv2d) { + int32_t depthwiseMultiplier = shaper[weight][3] / group; + input_indices.push_back(model_builder.AddOperandFromScalar(depthwiseMultiplier)); + } + int32_t fuse_code = model_builder.FindActivation(output); + input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); + // TODO support API 28 + input_indices.push_back(model_builder.AddOperandFromScalar(use_nchw)); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_dilations[1])); + input_indices.push_back(model_builder.AddOperandFromScalar(onnx_dilations[0])); + + int32_t operationCode; + if (conv2d) { + operationCode = ANEURALNETWORKS_CONV_2D; + shaper.Conv(input, weight, + onnx_pads, onnx_strides, onnx_dilations, + use_nchw, + output); + } else { // depthwise_conv2d + operationCode = ANEURALNETWORKS_DEPTHWISE_CONV_2D; + shaper.DepthwiseConv(input, weight, + onnx_pads, onnx_strides, onnx_dilations, + use_nchw, + output); + } + + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.AddOperation(operationCode, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); +} + void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -1368,6 +1844,11 @@ class CastOpBuilder : public BaseOpBuilder { bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) const override { + return 29; + } + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { return 29; @@ -1375,6 +1856,9 @@ class CastOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, @@ -1403,6 +1887,38 @@ bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } +void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + GraphNodeAttrHelper helper(node); + + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + bool output_is_nhwc = model_builder.IsOperandNHWC(input); + + auto to = helper.Get("to", 0); + Type type; + switch (to) { + case ONNX_NAMESPACE::TensorProto::FLOAT: + type = Type::TENSOR_FLOAT32; + break; + case ONNX_NAMESPACE::TensorProto::INT32: + type = Type::TENSOR_INT32; + break; + default: + ORT_THROW("Invalid cast to type: " + + std::to_string(to)); + } + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + shaper.Identity(input, output); + const OperandType output_operand_type(type, shaper[output]); + model_builder.AddOperation(ANEURALNETWORKS_CAST, input_indices, {output}, + {output_operand_type}, {output_is_nhwc}); +} + void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -1447,6 +1963,11 @@ class SoftMaxOpBuilder : public BaseOpBuilder { bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) const override { + return 29; + } + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) const override { return 29; @@ -1454,6 +1975,9 @@ class SoftMaxOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, @@ -1478,6 +2002,37 @@ bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } +void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + GraphNodeAttrHelper helper(node); + + auto input = node.InputDefs()[0]->Name(); + if (model_builder.IsOperandNHWC(input)) { + // We want to transpose nhwc operand back to nchw before softmax + const auto& nhwc_input = node.InputDefs()[0]->Name(); + if (!model_builder.GetNCHWOperand(nhwc_input, input)) { + input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); + TransposeNHWCToNCHW(model_builder, nhwc_input, input); + } + } + + const auto& output = node.OutputDefs()[0]->Name(); + float beta = 1.f; + int32_t axis = helper.Get("axis", 1); + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + input_indices.push_back(model_builder.AddOperandFromScalar(beta)); + input_indices.push_back(model_builder.AddOperandFromScalar(axis)); + + shaper.Identity(input, output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.AddOperation(ANEURALNETWORKS_SOFTMAX, input_indices, {output}, + {output_operand_type}, {false}); +} + void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -1517,8 +2072,31 @@ class IdentityOpBuilder : public BaseOpBuilder { private: void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; +void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + // Identity is not really going to do anything + // Just register the dimension and type, with same index and new name + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + bool output_is_nhwc = model_builder.IsOperandNHWC(input); + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); // input + + shaper.Identity(input, output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); +} + void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { // Identity is not really going to do anything @@ -1557,6 +2135,9 @@ class GemmOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, @@ -1689,6 +2270,59 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, } } +void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + const auto& op = node.OpType(); + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + GraphNodeAttrHelper helper(node); + + const auto& input1 = node.InputDefs()[0]->Name(); + const auto& input2 = node.InputDefs()[1]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + const auto transB = helper.Get("transB", 0); + + uint32_t input_2_idx; + if (transB == 0) { + input_2_idx = AddInitializerTransposed(model_builder, input2); + } else { + input_2_idx = operand_indices.at(input2); + } + + uint32_t bias_idx; + if (node.InputDefs().size() == 2) { + std::string bias = node.Name() + op + "_bias"; + const auto& B_type = operand_types.at(input2).type; + ModelBuilder::Shape bias_dimen = {shaper[input2][0]}; + if (B_type == Type::TENSOR_FLOAT32) { + float buffer[bias_dimen[0]]; + for (uint32_t i = 0; i < bias_dimen[0]; i++) { + buffer[i] = 0.f; + } + OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); + bias_idx = model_builder.AddOperandFromPersistMemoryBuffer( + bias, &buffer[0], operandType); + } else { + ORT_THROW("Unknown weight type " + TypeToStr(B_type)); + } + } else { + bias_idx = operand_indices.at(node.InputDefs()[2]->Name()); + } + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input1)); // A + input_indices.push_back(input_2_idx); // B + input_indices.push_back(bias_idx); // C + int32_t fuse_code = model_builder.FindActivation(output); + input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); + + shaper.FC(input1, input2, output); + const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); + model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices, {output}, + {output_operand_type}, {false}); +} + void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { const auto& op = node.op_type(); @@ -1748,14 +2382,34 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class UnaryOpBuilder : public BaseOpBuilder { private: + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, + const onnxruntime::Node& node) const override; + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) const override; - void AddToModelBuilderImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; +int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const onnxruntime::Node& node) const { + const auto& op(node.OpType()); + if (op == "Abs" || + op == "Exp" || + op == "Neg" || + op == "Sin" || + op == "Sqrt" || + op == "Log") { + return 29; + } + + return 27; +} + int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& node) const { const auto& op(node.op_type()); @@ -1771,6 +2425,47 @@ int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, return 27; } +void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + const auto& op_type(node.OpType()); + + const auto& input = node.InputDefs()[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + bool output_is_nhwc = model_builder.IsOperandNHWC(input); + + shaper.Identity(input, output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + + int32_t op_code; + if (op_type == "Abs") + op_code = ANEURALNETWORKS_ABS; + else if (op_type == "Exp") + op_code = ANEURALNETWORKS_EXP; + else if (op_type == "Floor") + op_code = ANEURALNETWORKS_FLOOR; + else if (op_type == "Log") + op_code = ANEURALNETWORKS_LOG; + else if (op_type == "Sigmoid") + op_code = ANEURALNETWORKS_LOGISTIC; + else if (op_type == "Neg") + op_code = ANEURALNETWORKS_NEG; + else if (op_type == "Sin") + op_code = ANEURALNETWORKS_SIN; + else if (op_type == "Sqrt") + op_code = ANEURALNETWORKS_SQRT; + else if (op_type == "Tanh") + op_code = ANEURALNETWORKS_TANH; + else { + ORT_THROW("UnaryOpBuilder, unknown op: " + op_type); + } + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); +} + void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); @@ -1826,6 +2521,9 @@ class ConcatOpBuilder : public BaseOpBuilder { void AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; }; bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, @@ -1850,6 +2548,77 @@ bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } +void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + GraphNodeAttrHelper helper(node); + + std::vector input_indices; + const auto& input0 = node.InputDefs()[0]->Name(); + bool all_input_have_same_layout = true; + bool output_is_nhwc = false; + const auto node_input_size = node.InputDefs().size(); + + // First we want to see if all the input are smae layout + for (size_t i = 0; i < node_input_size - 1; i++) { + all_input_have_same_layout = + all_input_have_same_layout && + model_builder.IsOperandNHWC(node.InputDefs()[i]->Name()) == + model_builder.IsOperandNHWC(node.InputDefs()[i + 1]->Name()); + } + + std::vector inputs; + inputs.reserve(node_input_size); + if (all_input_have_same_layout) { + // if all the inputs are of same layout, output will be the same layout + if (model_builder.IsOperandNHWC(input0)) { + output_is_nhwc = true; + } + + for (size_t i = 0; i < node_input_size; i++) { + auto input = node.InputDefs()[i]->Name(); + input_indices.push_back(operand_indices.at(input)); + inputs.push_back(input); + } + } else { + // if all the inputs are not same layout, + // will need transpos those nhwc tensors back to nchw + for (size_t i = 0; i < node_input_size; i++) { + auto input = node.InputDefs()[i]->Name(); + if (model_builder.IsOperandNHWC(input)) { + std::string nhwc_input = input; + input = model_builder.GetUniqueName(input + "_nhwc_to_nchw"); + TransposeNHWCToNCHW(model_builder, nhwc_input, input); + } + input_indices.push_back(operand_indices.at(input)); + inputs.push_back(input); + } + } + + int32_t axis = helper.Get("axis", 1); + int rank = shaper[input0].size(); + if (axis < 0) { // NNAPI does not support negative axis + axis = rank + axis; + } + + if (output_is_nhwc) { + ORT_ENFORCE(rank == 4, "nhwc is only on 4d shape, input " + input0 + + " has rank: " + std::to_string(rank)); + // we are using nhwc here, but the axis is in nwhw, need to transpose axis from nchw to nhwc + const uint32_t axis_nchw_to_nhwc[4]{0, 3, 1, 2}; + axis = axis_nchw_to_nhwc[axis]; + } + input_indices.push_back(model_builder.AddOperandFromScalar(axis)); + + const auto& output = node.OutputDefs()[0]->Name(); + shaper.Concat(inputs, axis, output); + const OperandType output_operand_type(operand_types.at(input0).type, shaper[output]); + model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, input_indices, {output}, + {output_operand_type}, {output_is_nhwc}); +} + void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) { auto& shaper(model_builder.GetShaper()); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index e338f63079248..c96f3d6467760 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -27,6 +27,10 @@ class IOpBuilder { // Add the operator to NNAPI model virtual void AddToModelBuilder(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) = 0; + + // Add the operator to NNAPI model + virtual void AddToModelBuilder(ModelBuilder& model_builder, + const onnxruntime::Node& node) = 0; }; // Generate a lookup table with IOpBuilder delegates From 711a311a89a064269f94918f521183707ea77c49 Mon Sep 17 00:00:00 2001 From: gwang Date: Tue, 30 Jun 2020 21:57:03 -0700 Subject: [PATCH 03/11] move the rest of model_builder to use graph instead of modelproto --- .../nnapi_builtin/builders/model_builder.cc | 147 ++++++++++-------- .../nnapi_builtin/builders/model_builder.h | 1 + .../nnapi_builtin/builders/op_builder.cc | 61 +++++++- .../nnapi/nnapi_builtin/builders/op_builder.h | 5 + 4 files changed, 141 insertions(+), 73 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index ed864d3d5b1f2..28e336f4670c3 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -40,21 +40,15 @@ bool ModelBuilder::IsNodeSupported( return false; } } -// bool ModelBuilder::IsNodeSupported( -// const ONNX_NAMESPACE::NodeProto& node) { -// if (auto* op_builder = GetOpBuilder(node)) { -// return op_builder->IsOpSupported(*this, node); -// } else { -// return false; -// } -// } bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, - const ONNX_NAMESPACE::ModelProto& model_proto) { + const onnxruntime::GraphViewer& graph_view) { if (!supported_node_vec.empty()) { if (supported_node_vec.size() == 1) { - const auto& node = model_proto.graph().node(supported_node_vec[0]); - const auto& op = node.op_type(); + const auto& node_indices = graph_view.GetNodesInTopologicalOrder(); + const auto* node(graph_view.GetNode(node_indices[supported_node_vec[0]])); + ORT_ENFORCE(nullptr != node, "node should not be null"); + const auto& op = node->OpType(); // It is not worth it to perform a single Reshape/Dropout/Identity operator // which is only copying the data in NNAPI // If this is the case, let it fall back @@ -67,7 +61,7 @@ bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, return true; } return false; -} // namespace nnapi +} std::vector> ModelBuilder::GetSupportedNodes() { std::vector> supported_node_vecs; @@ -82,23 +76,6 @@ std::vector> ModelBuilder::GetSupportedNodes() { #endif std::vector supported_node_vec; - // for (int i = 0; i < model_proto_.graph().node_size(); i++) { - // const auto& node(model_proto_.graph().node(i)); - // bool supported = IsNodeSupported(node); - // LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node.op_type() - // << "] index: [" << i - // << "] name: [" << node.name() - // << "] supported: [" << supported - // << "]"; - // if (supported) { - // supported_node_vec.push_back(i); - // } else { - // if (IsValidSupportedNodesVec(supported_node_vec, model_proto_)) { - // supported_node_vecs.push_back(supported_node_vec); - // supported_node_vec.clear(); - // } - // } - // } const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); for (size_t i = 0; i < node_indices.size(); i++) { const auto* node(graph_view_.GetNode(node_indices[i])); @@ -112,14 +89,14 @@ std::vector> ModelBuilder::GetSupportedNodes() { if (supported) { supported_node_vec.push_back(i); } else { - if (IsValidSupportedNodesVec(supported_node_vec, model_proto_)) { + if (IsValidSupportedNodesVec(supported_node_vec, graph_view_)) { supported_node_vecs.push_back(supported_node_vec); supported_node_vec.clear(); } } } - if (IsValidSupportedNodesVec(supported_node_vec, model_proto_)) + if (IsValidSupportedNodesVec(supported_node_vec, graph_view_)) supported_node_vecs.push_back(supported_node_vec); LOGS_DEFAULT(VERBOSE) << "Support vectors size is " << supported_node_vecs.size(); @@ -205,27 +182,32 @@ void ModelBuilder::GetTargetDevices() { } void ModelBuilder::GetAllInitializers() { - for (const auto& tensor : model_proto_.graph().initializer()) { - initializers_.emplace(tensor.name(), tensor); + for (const auto& pair : graph_view_.GetAllInitializedTensors()) { + ORT_ENFORCE(pair.second != nullptr, "Initializer is null for: " + pair.first); + initializers_.emplace(pair.first, *pair.second); } } void ModelBuilder::PreprocessInitializers() { - for (const auto& node : model_proto_.graph().node()) { - if (auto* opBuilder = GetOpBuilder(node)) { - opBuilder->AddInitializersToSkip(*this, node); + const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); + for (size_t i = 0; i < node_indices.size(); i++) { + const auto* node(graph_view_.GetNode(node_indices[i])); + ORT_ENFORCE(nullptr != node, "node should not be null"); + if (auto* opBuilder = GetOpBuilder(*node)) { + opBuilder->AddInitializersToSkip(*this, *node); } } } void ModelBuilder::RegisterInitializers() { // First pass to get all the stats of the initializers - auto initializer_size = model_proto_.graph().initializer_size(); + auto initializer_size = initializers_.size(); std::vector> initializers(initializer_size); size_t sizeAll = 0; - for (int i = 0; i < initializer_size; ++i) { - const auto& tensor = model_proto_.graph().initializer(i); + int i = 0; + for (const auto& pair : initializers_) { + const auto& tensor = pair.second; const auto& name = tensor.name(); if (Contains(skipped_initializers_, name)) continue; @@ -254,23 +236,24 @@ void ModelBuilder::RegisterInitializers() { const size_t size = operand_type.GetOperandBlobByteSize(); const size_t padded_size = GetPaddedByteSize(size); sizeAll += padded_size; - initializers[i] = std::make_tuple(index, size, padded_size); + initializers[i++] = std::make_tuple(index, size, padded_size); } // 2nd pass copies all the initializer data into NNAPI shared memory + i = 0; nnapi_model_->mem_initializers_ = std::make_unique(nnapi_, "mem_initializers_", sizeAll); // 2nd pass to copy all the initializers into shared memory size_t offset = 0; - for (int i = 0; i < initializer_size; ++i) { - const auto& tensor = model_proto_.graph().initializer(i); + for (const auto& pair : initializers_) { + const auto& tensor = pair.second; if (Contains(skipped_initializers_, tensor.name())) continue; uint32_t index; size_t size, padded_size; - std::tie(index, size, padded_size) = initializers[i]; + std::tie(index, size, padded_size) = initializers[i++]; const char* src = nullptr; if (tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { src = tensor.float_data().empty() @@ -286,9 +269,9 @@ void ModelBuilder::RegisterInitializers() { } void ModelBuilder::RegisterModelInputs() { - for (int32_t input_idx = 0; input_idx < model_proto_.graph().input_size(); input_idx++) { - const auto& input(model_proto_.graph().input(input_idx)); - std::string input_name = input.name(); + for (const auto* node_arg : graph_view_.GetInputs()) { + ORT_ENFORCE(node_arg != nullptr, "input cannot be null"); + const auto& input_name = node_arg->Name(); { // input should not be an initializer if (Contains(operands_, input_name)) @@ -298,15 +281,22 @@ void ModelBuilder::RegisterModelInputs() { continue; } + const auto* shape_proto = node_arg->Shape(); + ORT_ENFORCE(shape_proto != nullptr, "shape_proto cannot be null"); Shaper::Shape shape; - for (const auto& dim : input.type().tensor_type().shape().dim()) { + + for (const auto& dim : shape_proto->dim()) { // NNAPI uses 0 for dynamic dimension, which is the default value for dim.dim_value() shape.push_back(SafeInt(dim.dim_value())); } Type type = Type::TENSOR_FLOAT32; - if (input.type().tensor_type().has_elem_type()) { - switch (input.type().tensor_type().elem_type()) { + const auto* type_proto = node_arg->TypeAsProto(); + if (!type_proto || !type_proto->tensor_type().has_elem_type()) { + ORT_THROW("The input of graph doesn't have elem_type: " + + input_name); + } else { + switch (type_proto->tensor_type().elem_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: type = Type::TENSOR_FLOAT32; break; @@ -314,27 +304,24 @@ void ModelBuilder::RegisterModelInputs() { // TODO: support other type ORT_THROW("The input of graph doesn't have valid type, name: " + input_name + " type: " + - std::to_string(input.type().tensor_type().elem_type())); + std::to_string(type_proto->tensor_type().elem_type())); } - } else { - ORT_THROW("The input of graph doesn't have elem_type: " + - input_name); } OperandType operand_type(type, shape); shaper_.AddShape(input_name, operand_type.dimensions); auto index = AddNewOperand(input_name, operand_type, false /* is_nhwc */); - input_index_vec_.push_back(index); nnapi_model_->AddInput(input_name, operand_type); } -} // namespace nnapi +} void ModelBuilder::RegisterModelOutputs() { - for (int32_t output_idx = 0; output_idx < model_proto_.graph().output_size(); output_idx++) { - const auto& output(model_proto_.graph().output(output_idx)); - const std::string& output_name(output.name()); + for (const auto* node_arg : graph_view_.GetOutputs()) { + ORT_ENFORCE(node_arg != nullptr, "input cannot be null"); + const auto& output_name = node_arg->Name(); + if (!Contains(operands_, output_name)) { ORT_THROW("The output of graph is not registered" + output_name); } @@ -438,15 +425,6 @@ void ModelBuilder::AddOperations() { "Node not supported" + node->Name()); } } - - // for (const auto& node : model_proto_.graph().node()) { - // if (auto* opBuilder = GetOpBuilder(node)) { - // opBuilder->AddToModelBuilder(*this, node); - // } else { - // throw std::invalid_argument( - // "Node not supported" + node.name()); - // } - // } } void ModelBuilder::AddOperation(int op, const std::vector& input_indices, @@ -512,6 +490,41 @@ std::unique_ptr ModelBuilder::Compile() { return std::move(nnapi_model_); } +int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, + const NodeArg* output) { + ORT_ENFORCE(output != nullptr, "output cannot be null"); + int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; + for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) { + const auto& dst_node = it->GetNode(); + const auto* dst_input = dst_node.InputDefs()[it->GetDstArgIndex()]; + if (dst_node.OpType() == "Relu") { + if (output == dst_input) { + fuse_code = ANEURALNETWORKS_FUSED_RELU; + } + } else { + // if there is any other non-relu node using the output + // will add relu separately + if (output == dst_input) + return ANEURALNETWORKS_FUSED_NONE; + } + } + + // if output is a graph output, will add relu separately + if (fuse_code != ANEURALNETWORKS_FUSED_NONE) { + for (const auto* graph_output : graph_view_.GetOutputs()) { + if (output == graph_output) + return ANEURALNETWORKS_FUSED_NONE; + } + + LOGS_DEFAULT(VERBOSE) << "Node [" << node.Name() << "] type [" << node.OpType() + << "], fused the output [" << output->Name() << "]"; + + fused_activations_.insert(output->Name()); + } + + return fuse_code; +} + int32_t ModelBuilder::FindActivation(const std::string& output) { int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; const ONNX_NAMESPACE::NodeProto* activationNode{nullptr}; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index 579c78f9b2d3a..1907302ea672d 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -44,6 +44,7 @@ class ModelBuilder { // Find if an output has a fuseable activation (Relu) int32_t FindActivation(const std::string& output); + int32_t FindActivation(const onnxruntime::Node& node, const NodeArg* output); // Add an NNAPI scalar operand uint32_t AddOperandFromScalar(bool value); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index ee0be94d98746..f6a535d80cac9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -339,6 +339,9 @@ class BaseOpBuilder : public IOpBuilder { virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, const ONNX_NAMESPACE::NodeProto& /* node */) override {} + virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) override {} + bool IsOpSupported(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override final; @@ -569,7 +572,7 @@ void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } } - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); AddBinaryOperator(op_code, model_builder, input1, input2, fuse_code, output, output_is_nhwc); } @@ -643,7 +646,8 @@ void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); // skip this relu if it is some op's fuse output - if (Contains(model_builder.GetFusedActivations(), node.Name())) { + if (Contains(model_builder.GetFusedActivations(), input)) { + LOGS_DEFAULT(VERBOSE) << "Relu Node [" << node.Name() << "] fused"; model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); } else { std::vector input_indices; @@ -804,6 +808,9 @@ class ReshapeOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; @@ -823,6 +830,11 @@ void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, model_builder.AddInitializerToSkip(node.input(1)); } +void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); +} + bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { const auto& initializers(model_builder.GetInitializerTensors()); @@ -988,6 +1000,9 @@ class BatchNormalizationOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; @@ -1011,6 +1026,15 @@ void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_buil model_builder.AddInitializerToSkip(node.input(4)); //var } +void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + // skip everything except input0 for BatchNormalization + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); // scale + model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name()); // B + model_builder.AddInitializerToSkip(node.InputDefs()[3]->Name()); // mean + model_builder.AddInitializerToSkip(node.InputDefs()[4]->Name()); //var +} + bool BatchNormalizationOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { if (node.OutputDefs().size() != 1) { @@ -1140,7 +1164,7 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil output_is_nhwc); // Add - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); AddBinaryOperator(ANEURALNETWORKS_ADD, model_builder, tensor_imm_product_name, tensor_b_name, @@ -1411,7 +1435,7 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, static_cast(shaper[input][2])}; } - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); std::vector input_indices; input_indices.push_back(operand_indices.at(input)); input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[1])); @@ -1514,6 +1538,9 @@ class ConvOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; @@ -1534,6 +1561,12 @@ void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, model_builder.AddInitializerToSkip(node.input(1)); } +void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + // skip the weight for conv as we need to transpose + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); +} + bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { GraphNodeAttrHelper helper(node); @@ -1685,7 +1718,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, int32_t depthwiseMultiplier = shaper[weight][3] / group; input_indices.push_back(model_builder.AddOperandFromScalar(depthwiseMultiplier)); } - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); // TODO support API 28 input_indices.push_back(model_builder.AddOperandFromScalar(use_nchw)); @@ -2126,6 +2159,9 @@ class GemmOpBuilder : public BaseOpBuilder { void AddInitializersToSkip(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; + private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) override; @@ -2270,6 +2306,19 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, } } +void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + const auto& op = node.OpType(); + if (op == "MatMul") { + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + } else if (op == "Gemm") { + GraphNodeAttrHelper helper(node); + const auto transB = helper.Get("transB", 0); + if (transB == 0) + model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + } +} + void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { const auto& op = node.OpType(); @@ -2314,7 +2363,7 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, input_indices.push_back(operand_indices.at(input1)); // A input_indices.push_back(input_2_idx); // B input_indices.push_back(bias_idx); // C - int32_t fuse_code = model_builder.FindActivation(output); + int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); shaper.FC(input1, input2, output); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index c96f3d6467760..f8d3a46e45f3c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -24,6 +24,11 @@ class IOpBuilder { virtual void AddInitializersToSkip(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) = 0; + // Check if the initializers of this operator need preprocess + // which will not be copied + virtual void AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) = 0; + // Add the operator to NNAPI model virtual void AddToModelBuilder(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node) = 0; From 4b4ed7849e2fdfbc3d51e5bfd1ec440aef18ee13 Mon Sep 17 00:00:00 2001 From: gwang Date: Tue, 30 Jun 2020 22:45:13 -0700 Subject: [PATCH 04/11] remove redundant code --- .../nnapi/nnapi_builtin/builders/helper.cc | 72 + .../nnapi/nnapi_builtin/builders/helper.h | 21 +- .../nnapi_builtin/builders/model_builder.cc | 47 +- .../nnapi_builtin/builders/model_builder.h | 8 +- .../builders/node_attr_helper.cc | 163 --- .../nnapi_builtin/builders/node_attr_helper.h | 44 - .../nnapi_builtin/builders/op_builder.cc | 1296 +---------------- .../nnapi/nnapi_builtin/builders/op_builder.h | 12 - .../nnapi_builtin/nnapi_execution_provider.cc | 34 +- .../nnapi_lib/NeuralNetworksWrapper.h | 2 - 10 files changed, 138 insertions(+), 1561 deletions(-) create mode 100644 onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc delete mode 100644 onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc delete mode 100644 onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc new file mode 100644 index 0000000000000..f0c9f69126e40 --- /dev/null +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -0,0 +1,72 @@ +// +// Created by daquexian on 8/3/18. +// + +#include +#include +#include +#include +#include + +#include "helper.h" + +using std::string; +using std::vector; + +NodeAttrHelper::NodeAttrHelper(const onnxruntime::Node& node) + : node_attributes_(node.GetAttributes()) {} + +float NodeAttrHelper::Get(const std::string& key, float def_val) const { + if (HasAttr(key)) + return node_attributes_.at(key).f(); + + return def_val; +} + +int32_t NodeAttrHelper::Get(const std::string& key, int32_t def_val) const { + if (HasAttr(key)) + return SafeInt(node_attributes_.at(key).i()); + + return def_val; +} + +string NodeAttrHelper::Get(const std::string& key, const string& def_val) const { + if (HasAttr(key)) + return node_attributes_.at(key).s(); + + return def_val; +} + +vector NodeAttrHelper::Get(const std::string& key, const vector& def_val) const { + if (HasAttr(key)) { + const auto& attr(node_attributes_.at(key)); + std::vector v; + v.reserve(static_cast(attr.ints_size())); + for (int j = 0; j < attr.ints_size(); j++) { + int64_t val = attr.ints(j); + v.push_back(SafeInt(val)); + } + return v; + } + + return def_val; +} + +vector NodeAttrHelper::Get(const std::string& key, const vector& def_val) const { + if (HasAttr(key)) { + const auto& attr(node_attributes_.at(key)); + std::vector v; + v.reserve(static_cast(attr.ints_size())); + for (int j = 0; j < attr.ints_size(); j++) { + v.push_back(attr.floats(j)); + } + + return v; + } + + return def_val; +} + +bool NodeAttrHelper::HasAttr(const std::string& key) const { + return Contains(node_attributes_, key); +} diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index 4faf37b10cb63..6d8662beefa3f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -3,7 +3,7 @@ // #pragma once -#include +#include #include #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h" @@ -61,3 +61,22 @@ inline std::string GetErrorCause(int error_code) { return "Unknown error code: " + std::to_string(error_code); } } + +/** + * Wrapping onnxruntime::Node for retrieving attribute values + */ +class NodeAttrHelper { + public: + NodeAttrHelper(const onnxruntime::Node& proto); + + float Get(const std::string& key, float def_val) const; + int32_t Get(const std::string& key, int32_t def_val) const; + std::vector Get(const std::string& key, const std::vector& def_val) const; + std::vector Get(const std::string& key, const std::vector& def_val) const; + std::string Get(const std::string& key, const std::string& def_val) const; + + bool HasAttr(const std::string& key) const; + + private: + const onnxruntime::NodeAttributes& node_attributes_; +}; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 28e336f4670c3..20670068fbbb0 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -2,12 +2,13 @@ // Licensed under the MIT License. #include +#include +#include -#include "core/common/safeint.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" #include "helper.h" #include "model_builder.h" -#include "node_attr_helper.h" +#include "op_builder.h" namespace onnxruntime { namespace nnapi { @@ -21,9 +22,8 @@ const float* GetTensorFloatDataA(const ONNX_NAMESPACE::TensorProto& tensor) { : tensor.float_data().data(); } -ModelBuilder::ModelBuilder(const ONNX_NAMESPACE::ModelProto& model_proto, - const onnxruntime::GraphViewer& graph_view) - : nnapi_(NnApiImplementation()), model_proto_(model_proto), graph_view_(graph_view) { +ModelBuilder::ModelBuilder(const onnxruntime::GraphViewer& graph_view) + : nnapi_(NnApiImplementation()), graph_view_(graph_view) { GetAllInitializers(); op_builders_ = CreateOpBuilders(); } @@ -525,43 +525,6 @@ int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, return fuse_code; } -int32_t ModelBuilder::FindActivation(const std::string& output) { - int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; - const ONNX_NAMESPACE::NodeProto* activationNode{nullptr}; - std::string node_name; - for (const auto& _node : model_proto_.graph().node()) { - if (_node.op_type() == "Relu" && output == _node.input(0)) { - fuse_code = ANEURALNETWORKS_FUSED_RELU; - activationNode = &_node; - } - } - - if (fuse_code != ANEURALNETWORKS_FUSED_NONE) { - for (const auto& _node : model_proto_.graph().node()) { - if (&_node == activationNode) - continue; - - // if there is any other node using the output - // will add relu separately - for (const auto& node_input : _node.input()) { - if (output == node_input) - return ANEURALNETWORKS_FUSED_NONE; - } - } - - // if output is a graph output - // will add relu separately - for (const auto& model_output : model_proto_.graph().output()) { - if (model_output.name() == output) - return ANEURALNETWORKS_FUSED_NONE; - } - - fused_activations_.insert(activationNode->name()); - } - - return fuse_code; -} - IOpBuilder* ModelBuilder::GetOpBuilder(const ONNX_NAMESPACE::NodeProto& node) { if (!Contains(op_builders_, node.op_type())) return nullptr; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index 1907302ea672d..e8ff55f9d8023 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -8,12 +8,13 @@ #include #include "core/providers/nnapi/nnapi_builtin/model.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" -#include "op_builder.h" #include "shaper.h" namespace onnxruntime { namespace nnapi { +class IOpBuilder; + class ModelBuilder { public: using Shape = Shaper::Shape; @@ -27,7 +28,7 @@ class ModelBuilder { CPU_ONLY, // use CPU only }; - ModelBuilder(const ONNX_NAMESPACE::ModelProto& model_proto, const onnxruntime::GraphViewer& graph_view); + ModelBuilder(const onnxruntime::GraphViewer& graph_view); ~ModelBuilder() = default; std::vector> GetSupportedNodes(); @@ -43,7 +44,6 @@ class ModelBuilder { const std::vector& is_nhwc_vec); // Find if an output has a fuseable activation (Relu) - int32_t FindActivation(const std::string& output); int32_t FindActivation(const onnxruntime::Node& node, const NodeArg* output); // Add an NNAPI scalar operand @@ -95,7 +95,6 @@ class ModelBuilder { const ONNX_NAMESPACE::TensorProto&>& GetInitializerTensors() const { return initializers_; } - const ONNX_NAMESPACE::ModelProto& GetOnnxModel() const { return model_proto_; } const onnxruntime::Graph& GetOnnxGraph() const { return graph_view_.GetGraph(); } void RegisterNHWCOperand(const std::string& name); @@ -112,7 +111,6 @@ class ModelBuilder { private: const NnApi* nnapi_{nullptr}; - const ONNX_NAMESPACE::ModelProto& model_proto_; const onnxruntime::GraphViewer& graph_view_; std::unique_ptr nnapi_model_; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc deleted file mode 100644 index 799686c38a5dd..0000000000000 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.cc +++ /dev/null @@ -1,163 +0,0 @@ -// -// Created by daquexian on 8/3/18. -// - -#include -#include -#include - -#include "core/common/safeint.h" -#include "helper.h" -#include "node_attr_helper.h" - -using std::string; -using std::vector; - -NodeAttrHelper::NodeAttrHelper(const ONNX_NAMESPACE::NodeProto& proto) : node_(proto) { -} - -float NodeAttrHelper::Get(const std::string& key, float def_val) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - return attr.f(); - } - } - - return def_val; -} - -int32_t NodeAttrHelper::Get(const std::string& key, int32_t def_val) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - int64_t val = attr.i(); - return SafeInt(val); - } - } - - return def_val; -} - -string NodeAttrHelper::Get(const std::string& key, const string& def_val) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - return attr.s(); - } - } - - return def_val; -} - -vector NodeAttrHelper::Get(const std::string& key, const vector& def_val) { - if (!HasAttr(key)) { - return def_val; - } - - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - std::vector v; - v.reserve(static_cast(attr.ints_size())); - for (int j = 0; j < attr.ints_size(); j++) { - int64_t val = attr.ints(j); - v.push_back(SafeInt(val)); - } - return v; - } - } - - return def_val; -} - -vector NodeAttrHelper::Get(const std::string& key, - const vector& def_val) { - if (!HasAttr(key)) { - return def_val; - } - - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - std::vector v; - v.reserve(static_cast(attr.floats_size())); - for (int j = 0; j < attr.floats_size(); j++) { - v.push_back(attr.floats(j)); - } - - return v; - } - } - - return def_val; -} - -bool NodeAttrHelper::HasAttr(const std::string& key) { - for (int i = 0; i < node_.attribute_size(); i++) { - const ONNX_NAMESPACE::AttributeProto& attr = node_.attribute(i); - if (attr.name() == key) { - return true; - } - } - - return false; -} - -GraphNodeAttrHelper::GraphNodeAttrHelper(const onnxruntime::Node& node) - : node_attributes_(node.GetAttributes()) {} - -float GraphNodeAttrHelper::Get(const std::string& key, float def_val) const { - if (HasAttr(key)) - return node_attributes_.at(key).f(); - - return def_val; -} - -int32_t GraphNodeAttrHelper::Get(const std::string& key, int32_t def_val) const { - if (HasAttr(key)) - return SafeInt(node_attributes_.at(key).i()); - - return def_val; -} - -string GraphNodeAttrHelper::Get(const std::string& key, const string& def_val) const { - if (HasAttr(key)) - return node_attributes_.at(key).s(); - - return def_val; -} - -vector GraphNodeAttrHelper::Get(const std::string& key, const vector& def_val) const { - if (HasAttr(key)) { - const auto& attr(node_attributes_.at(key)); - std::vector v; - v.reserve(static_cast(attr.ints_size())); - for (int j = 0; j < attr.ints_size(); j++) { - int64_t val = attr.ints(j); - v.push_back(SafeInt(val)); - } - return v; - } - - return def_val; -} - -vector GraphNodeAttrHelper::Get(const std::string& key, const vector& def_val) const { - if (HasAttr(key)) { - const auto& attr(node_attributes_.at(key)); - std::vector v; - v.reserve(static_cast(attr.ints_size())); - for (int j = 0; j < attr.ints_size(); j++) { - v.push_back(attr.floats(j)); - } - - return v; - } - - return def_val; -} - -bool GraphNodeAttrHelper::HasAttr(const std::string& key) const { - return Contains(node_attributes_, key); -} diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h deleted file mode 100644 index 7e18b0d87764c..0000000000000 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/node_attr_helper.h +++ /dev/null @@ -1,44 +0,0 @@ -// -// Created by daquexian on 8/3/18. -// - -#pragma once - -#include -#include -#include - -/** - * Wrapping onnx::NodeProto for retrieving attribute values - */ -class NodeAttrHelper { - public: - NodeAttrHelper(const ONNX_NAMESPACE::NodeProto& proto); - - float Get(const std::string& key, float def_val); - int32_t Get(const std::string& key, int32_t def_val); - std::vector Get(const std::string& key, const std::vector& def_val); - std::vector Get(const std::string& key, const std::vector& def_val); - std::string Get(const std::string& key, const std::string& def_val); - - bool HasAttr(const std::string& key); - - private: - const ONNX_NAMESPACE::NodeProto& node_; -}; - -class GraphNodeAttrHelper { - public: - GraphNodeAttrHelper(const onnxruntime::Node& proto); - - float Get(const std::string& key, float def_val) const; - int32_t Get(const std::string& key, int32_t def_val) const; - std::vector Get(const std::string& key, const std::vector& def_val) const; - std::vector Get(const std::string& key, const std::vector& def_val) const; - std::string Get(const std::string& key, const std::string& def_val) const; - - bool HasAttr(const std::string& key) const; - - private: - const onnxruntime::NodeAttributes& node_attributes_; -}; \ No newline at end of file diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index f6a535d80cac9..d9edd6cdc37ed 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -2,12 +2,11 @@ // Licensed under the MIT License. #include +#include #include -#include "core/common/safeint.h" #include "helper.h" #include "model_builder.h" -#include "node_attr_helper.h" #include "op_builder.h" namespace onnxruntime { @@ -124,32 +123,6 @@ void AddBinaryOperator(int32_t op_type, model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -int GetType(const ONNX_NAMESPACE::ModelProto& model_proto, - const std::string& name) { - int invalid_type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; - for (const auto& input : model_proto.graph().input()) { - if (input.name() != name) - continue; - - return input.type().tensor_type().elem_type(); - } - - for (const auto& value_info : model_proto.graph().value_info()) { - if (value_info.name() != name) - continue; - - if (!value_info.has_type()) { - return invalid_type; - } else if (!value_info.type().has_tensor_type()) { - return invalid_type; - } - - return value_info.type().tensor_type().elem_type(); - } - - return invalid_type; -} - bool GetType(const onnxruntime::Node& node, int32_t& type) { type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; auto* type_proto = node.InputDefs()[0]->TypeAsProto(); @@ -174,55 +147,6 @@ Shaper::Shape GetShape(const ONNX_NAMESPACE::TensorShapeProto* shape_proto) { return shape; } -Shaper::Shape GetShape(const ONNX_NAMESPACE::ModelProto& model_proto, - const std::string& name) { - Shaper::Shape empty_shape; - for (const auto& input : model_proto.graph().input()) { - if (input.name() != name) - continue; - - Shaper::Shape shape; - for (const auto& dim : input.type().tensor_type().shape().dim()) - shape.push_back(dim.dim_value()); - - return shape; - } - - for (const auto& tensor : model_proto.graph().initializer()) { - if (tensor.name() != name) - continue; - - Shaper::Shape shape; - for (auto dim : tensor.dims()) - shape.push_back(SafeInt(dim)); - - return shape; - } - - for (const auto& value_info : model_proto.graph().value_info()) { - if (value_info.name() != name) - continue; - - if (!value_info.has_type()) { - return empty_shape; - } else if (!value_info.type().has_tensor_type()) { - return empty_shape; - } else if (!value_info.type().tensor_type().has_shape()) { - return empty_shape; - } else if (value_info.type().tensor_type().shape().dim_size() == 0) { - return empty_shape; - } - - Shaper::Shape shape; - for (const auto& dim : value_info.type().tensor_type().shape().dim()) - shape.push_back(dim.dim_value()); - - return shape; - } - - return empty_shape; -} - enum DataLayout { L_0231 = 0, L_1230 = 1, @@ -336,69 +260,29 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, class BaseOpBuilder : public IOpBuilder { public: virtual ~BaseOpBuilder() = default; - virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) override {} - virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, const onnxruntime::Node& /* node */) override {} - bool IsOpSupported(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override final; - bool IsOpSupported(ModelBuilder& model_builder, const onnxruntime::Node& node) override final; - void AddToModelBuilder(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override final; - void AddToModelBuilder(ModelBuilder& model_builder, const onnxruntime::Node& node) override final; protected: - virtual bool IsOpSupportedImpl( - ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); - virtual bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node); - virtual int32_t GetMinSupportedSdkVer( - ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const { return 27; } - virtual int32_t GetMinSupportedSdkVer( ModelBuilder& /* model_builder */, const onnxruntime::Node& /* node */) const { return 27; } - virtual bool HasSupportedInputs(ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); - virtual bool HasSupportedInputs(const onnxruntime::Node& node); - virtual void AddToModelBuilderImpl( - ModelBuilder& model_builder, const ONNX_NAMESPACE::NodeProto& node); - virtual void AddToModelBuilderImpl( ModelBuilder& model_builder, const onnxruntime::Node& node); }; -bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { -#ifdef __ANDROID__ - int32_t android_sdk_ver = model_builder.GetAndroidSdkVer(); - int32_t required_sdk_ver = GetMinSupportedSdkVer(model_builder, node); - if (required_sdk_ver > android_sdk_ver) { - LOGS_DEFAULT(VERBOSE) << "Current Android API level [" << android_sdk_ver - << "], Operator [" << node.op_type() - << "] is only supported on API >" << required_sdk_ver; - return false; - } -#endif - - if (!HasSupportedInputs(model_builder, node)) - return false; - - return IsOpSupportedImpl(model_builder, node); -} - bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, const onnxruntime::Node& node) { #ifdef __ANDROID__ @@ -418,22 +302,6 @@ bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, return IsOpSupportedImpl(model_builder, node); } -bool BaseOpBuilder::HasSupportedInputs( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - // We only check the type of input 0 by default - // specific op builder can override this - auto input_type = GetType(model_builder.GetOnnxModel(), node.input(0)); - if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - LOGS_DEFAULT(VERBOSE) << "[" << node.op_type() - << "] Input type: [" << input_type - << "] is not supported for now"; - return false; - } - - return true; -} - bool BaseOpBuilder::HasSupportedInputs(const onnxruntime::Node& node) { // We only check the type of input 0 by default // specific op builder can override this @@ -451,11 +319,6 @@ bool BaseOpBuilder::HasSupportedInputs(const onnxruntime::Node& node) { return true; } -bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) { - return true; -} - bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const onnxruntime::Node& /* node */) { return true; @@ -471,21 +334,6 @@ void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, << "] type: [" << node.OpType() << "] was added"; } -void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - ORT_ENFORCE(IsOpSupported(model_builder, node), - "Unsupported operator " + node.op_type()); - - AddToModelBuilderImpl(model_builder, node); - LOGS_DEFAULT(VERBOSE) << "Operator name: [" << node.name() - << "] type: [" << node.op_type() << "] was added"; -} - -void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) { - ORT_NOT_IMPLEMENTED("Unsupported operator " + node.op_type()); -} - void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, const onnxruntime::Node& node) { ORT_NOT_IMPLEMENTED("Unsupported operator " + node.OpType()); @@ -497,16 +345,10 @@ void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, class BinaryOpBuilder : public BaseOpBuilder { private: - int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) const override; - int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const onnxruntime::Node& node) const override; private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -521,16 +363,6 @@ int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */ return 27; } -int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) const { - const auto& op(node.op_type()); - if (op == "Sub" || op == "Div") { - return 28; - } - - return 27; -} - void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { const auto& op(node.OpType()); @@ -576,59 +408,12 @@ void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, AddBinaryOperator(op_code, model_builder, input1, input2, fuse_code, output, output_is_nhwc); } -void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op(node.op_type()); - int32_t op_code; - if (op == "Add") - op_code = ANEURALNETWORKS_ADD; - else if (op == "Sub") - op_code = ANEURALNETWORKS_SUB; - else if (op == "Mul") - op_code = ANEURALNETWORKS_MUL; - else if (op == "Div") - op_code = ANEURALNETWORKS_DIV; - else { - ORT_THROW("UnaryOpBuilder, unknown op: " + op); - } - std::string input1 = node.input(0); - std::string input2 = node.input(1); - bool input1_is_nhwc = model_builder.IsOperandNHWC(input1); - bool input2_is_nhwc = model_builder.IsOperandNHWC(input2); - bool output_is_nhwc = false; - - if (input1_is_nhwc == input2_is_nhwc) { - output_is_nhwc = input1_is_nhwc; - } else if (input1_is_nhwc) { - // need transpsoe input1 back to nchw - const auto& nhwc_input = node.input(0); - if (!model_builder.GetNCHWOperand(nhwc_input, input1)) { - input1 = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); - TransposeNHWCToNCHW(model_builder, nhwc_input, input1); - } - } else { // input2_is_nhwc - // need transpsoe input2 back to nchw - const auto& nhwc_input = node.input(1); - if (!model_builder.GetNCHWOperand(nhwc_input, input2)) { - input2 = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); - TransposeNHWCToNCHW(model_builder, nhwc_input, input2); - } - } - - const auto& output = node.output(0); - int32_t fuse_code = model_builder.FindActivation(output); - AddBinaryOperator(op_code, model_builder, input1, input2, fuse_code, output, output_is_nhwc); -} - #pragma endregion #pragma region op_relu class ReluOpBuilder : public BaseOpBuilder { private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -656,37 +441,12 @@ void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } } -void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - - const auto& input = node.input(0); - const auto& output = node.output(0); - bool output_is_nhwc = model_builder.IsOperandNHWC(input); - shaper.Identity(input, output); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - - // skip this relu if it is some op's fuse output - if (Contains(model_builder.GetFusedActivations(), node.name())) { - model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); - } else { - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); - model_builder.AddOperation(ANEURALNETWORKS_RELU, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); - } -} - #pragma endregion op_relu #pragma region op_transpose class TransposeOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; @@ -695,14 +455,6 @@ class TransposeOpBuilder : public BaseOpBuilder { return 28; } - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { - return 28; - } - - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -719,58 +471,12 @@ bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); - if (input_size > 4) { - LOGS_DEFAULT(VERBOSE) << "Transpose only supports up to 4d shape, input is " - << input_size << "d shape"; - return false; - } - - return true; -} - void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); auto input = node.InputDefs()[0]->Name(); const auto& output = node.OutputDefs()[0]->Name(); - GraphNodeAttrHelper helper(node); - vector perm = helper.Get("perm", vector()); - auto input_dims = shaper[input].size(); - if (perm.empty()) { - for (int32_t i = input_dims - 1; i >= 0; i--) - perm.push_back(i); - } else { - ORT_ENFORCE(perm.size() == input_dims, "Perm and input should have same dimension"); - } - - if (model_builder.IsOperandNHWC(input)) { - ORT_ENFORCE(input_dims == 4, "Only 4D shape can be nhwc"); - - // we are using nhwc here, but the axis is in nchw, need to transpose axis from nchw to nhwc - const int32_t axis_nchw_to_nhwc[4]{0, 3, 1, 2}; - for (size_t i = 0; i < perm.size(); i++) - perm[i] = axis_nchw_to_nhwc[perm[i]]; - } - - std::string perm_name = model_builder.GetUniqueName(node.Name() + input + "perm"); - - // It is possible this onnx transpose operator can be nchw->nhwc, but so far I don't see - // any scenario will do this since onnx is nchw only, assume the output is always not nhwc - // even it is, there will be extra transpose in the onnx model to convert it back to nchw - // before conv/pool/... operators - AddTransposeOperator(model_builder, input, perm_name, perm, output, false /* is_nhwc */); -} - -void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - - auto input = node.input(0); - const auto& output = node.output(0); NodeAttrHelper helper(node); vector perm = helper.Get("perm", vector()); auto input_dims = shaper[input].size(); @@ -790,7 +496,7 @@ void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, perm[i] = axis_nchw_to_nhwc[perm[i]]; } - std::string perm_name = model_builder.GetUniqueName(node.name() + input + "perm"); + std::string perm_name = model_builder.GetUniqueName(node.Name() + input + "perm"); // It is possible this onnx transpose operator can be nchw->nhwc, but so far I don't see // any scenario will do this since onnx is nchw only, assume the output is always not nhwc @@ -805,31 +511,17 @@ void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ReshapeOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; -void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - model_builder.AddInitializerToSkip(node.input(1)); -} - void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) { model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); @@ -866,39 +558,6 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -bool ReshapeOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& initializers(model_builder.GetInitializerTensors()); - if (!Contains(initializers, node.input(1))) { - LOGS_DEFAULT(VERBOSE) << "New shape of reshape must be known"; - return false; - } - - const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); - if (input_size > 4) { - LOGS_DEFAULT(VERBOSE) << "Reshape only supports up to 4d shape, input is " - << input_size << "d shape"; - return false; - } - - const auto& shape_tensor = initializers.at(node.input(1)); - const int64_t* rawShape = GetTensorInt64Data(shape_tensor); - const auto size = SafeInt(shape_tensor.dims()[0]); - const auto input_shape = GetShape(model_builder.GetOnnxModel(), node.input(0)); - - for (uint32_t i = 0; i < size; i++) { - // NNAPI reshape does not support 0 as dimension - if (rawShape[i] == 0 && i < input_shape.size() && input_shape[i] == 0) { - LOGS_DEFAULT(VERBOSE) - << "Reshape doesn't suppport 0 reshape dimension on a dynamic dimension"; - return false; - } - } - - return true; -} - void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); @@ -945,86 +604,22 @@ void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, {output}, {output_operand_type}, {false}); } -void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - const auto& initializers(model_builder.GetInitializerTensors()); +#pragma endregion op_reshape - auto input = node.input(0); +#pragma region op_batchnormalization - if (model_builder.IsOperandNHWC(input)) { - // We want to transpose nhwc operand back to nchw before reshape - const auto& nhwc_input = node.input(0); - if (!model_builder.GetNCHWOperand(nhwc_input, input)) { - input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); - TransposeNHWCToNCHW(model_builder, nhwc_input, input); - } - } +class BatchNormalizationOpBuilder : public BaseOpBuilder { + public: + void AddInitializersToSkip(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; - const auto& output = node.output(0); - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); // input + private: + bool IsOpSupportedImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; - const auto& shape_tensor = initializers.at(node.input(1)); - const int64_t* rawShape = GetTensorInt64Data(shape_tensor); - const auto size = SafeInt(shape_tensor.dims()[0]); - - ModelBuilder::Shape input_shape = shaper[input]; - std::vector shape(size); - for (uint32_t i = 0; i < size; i++) { - int32_t dim = SafeInt(rawShape[i]); - // NNAPI reshape does not support 0 as dimension - shape[i] = dim == 0 ? input_shape[i] : dim; - } - - ModelBuilder::Shape shape_dimen = {size}; - std::string shape_name = model_builder.GetUniqueName(node.name() + input + "newshape"); - OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen); - uint32_t shape_idx = model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type); - input_indices.push_back(shape_idx); - - shaper.Reshape(input, shape, output); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices, - {output}, {output_operand_type}, {false}); -} - -#pragma endregion op_reshape - -#pragma region op_batchnormalization - -class BatchNormalizationOpBuilder : public BaseOpBuilder { - public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - - void AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; - - private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; - - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; -}; - -void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - // skip everything except input0 for BatchNormalization - model_builder.AddInitializerToSkip(node.input(1)); // scale - model_builder.AddInitializerToSkip(node.input(2)); // B - model_builder.AddInitializerToSkip(node.input(3)); // mean - model_builder.AddInitializerToSkip(node.input(4)); //var -} + void AddToModelBuilderImpl(ModelBuilder& model_builder, + const onnxruntime::Node& node) override; +}; void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) { @@ -1068,46 +663,12 @@ bool BatchNormalizationOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -bool BatchNormalizationOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - if (node.output_size() != 1) { - LOGS_DEFAULT(VERBOSE) << "Your onnx model may be in training mode, please export " - "it in test mode."; - return false; - } - - const auto& initializers(model_builder.GetInitializerTensors()); - const auto& scale_name = node.input(1); - const auto& b_name = node.input(2); - const auto& mean_name = node.input(3); - const auto& var_name = node.input(4); - if (!Contains(initializers, scale_name)) { - LOGS_DEFAULT(VERBOSE) << "Scale of BN must be known"; - return false; - } - if (!Contains(initializers, b_name)) { - LOGS_DEFAULT(VERBOSE) << "B of BN must be known"; - return false; - } - if (!Contains(initializers, mean_name)) { - LOGS_DEFAULT(VERBOSE) << "Mean of BN must be known"; - return false; - } - if (!Contains(initializers, var_name)) { - LOGS_DEFAULT(VERBOSE) << "Var of BN must be known"; - return false; - } - - return true; -} - void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); // For reshape we are not really doing anything but // register a new operand with new shape @@ -1173,86 +734,12 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil output_is_nhwc); } -void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_types(model_builder.GetOperandTypes()); - const auto& initializers(model_builder.GetInitializerTensors()); - NodeAttrHelper helper(node); - - // For reshape we are not really doing anything but - // register a new operand with new shape - const auto& input = node.input(0); - const auto& output = node.output(0); - - const auto& scale_tensor = initializers.at(node.input(1)); - const auto& bias_tensor = initializers.at(node.input(2)); - const auto& mean_tensor = initializers.at(node.input(3)); - const auto& var_tensor = initializers.at(node.input(4)); - const auto eps = helper.Get("epsilon", 1e-5f); - - const auto size = SafeInt(scale_tensor.dims()[0]); - vector a, b; - a.reserve(size); - b.reserve(size); - - const float* scale_data = GetTensorFloatData(scale_tensor); - const float* bias_data = GetTensorFloatData(bias_tensor); - const float* mean_data = GetTensorFloatData(mean_tensor); - const float* var_data = GetTensorFloatData(var_tensor); - - for (int64_t i = 0; i < size; i++) { - a.push_back(scale_data[i] / sqrt(var_data[i] + eps)); - b.push_back((scale_data[i] * -mean_data[i]) / sqrt(var_data[i] + eps) + - bias_data[i]); - } - - const auto tensor_a_name = model_builder.GetUniqueName(node.name() + input + "_imm_a"); - const auto tensor_b_name = model_builder.GetUniqueName(node.name() + input + "_imm_b"); - const auto tensor_imm_product_name = model_builder.GetUniqueName(node.name() + input + "_imm_mul"); - ModelBuilder::Shape tensor_a_dimen; - - bool input_is_nhwc = model_builder.IsOperandNHWC(input); - bool output_is_nhwc = input_is_nhwc; - if (input_is_nhwc) - tensor_a_dimen = {size}; - else // input is nchw - tensor_a_dimen = {size, 1, 1}; // {C, H, W} - - shaper.AddShape(tensor_a_name, tensor_a_dimen); - shaper.AddShape(tensor_b_name, tensor_a_dimen); - const OperandType operandType_a(operand_types.at(input).type, tensor_a_dimen); - model_builder.AddOperandFromPersistMemoryBuffer(tensor_a_name, a.data(), operandType_a); - const OperandType operandType_b(operand_types.at(input).type, tensor_a_dimen); - model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), operandType_b); - - // Mul - AddBinaryOperator(ANEURALNETWORKS_MUL, - model_builder, - input, tensor_a_name, - ANEURALNETWORKS_FUSED_NONE, - tensor_imm_product_name, - output_is_nhwc); - - // Add - int32_t fuse_code = model_builder.FindActivation(output); - AddBinaryOperator(ANEURALNETWORKS_ADD, - model_builder, - tensor_imm_product_name, tensor_b_name, - fuse_code, - output, - output_is_nhwc); -} - #pragma endregion op_batchnormalization #pragma region op_pool class PoolOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; @@ -1261,14 +748,6 @@ class PoolOpBuilder : public BaseOpBuilder { return 28; } - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { - return 29; - } - - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -1277,7 +756,7 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const onnxruntime::Node& node) { const auto& op = node.OpType(); if (op == "AveragePool" || op == "MaxPool") { - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); const auto count_include_pad = helper.Get("count_include_pad", 0); if (count_include_pad == 1) { @@ -1329,69 +808,13 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); - if (op == "AveragePool" || op == "MaxPool") { - NodeAttrHelper helper(node); - - const auto count_include_pad = helper.Get("count_include_pad", 0); - if (count_include_pad == 1) { - LOGS_DEFAULT(VERBOSE) << "count_include_pad == 1 is not supported"; - return false; - } - - const auto storage_order = helper.Get("storage_order", 0); - if (storage_order == 1) { - LOGS_DEFAULT(VERBOSE) << "storage_order == 1 is not supported"; - return false; - } - - if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { - LOGS_DEFAULT(VERBOSE) << "auto_pad is not supported"; - return false; - } - - if (helper.Get("kernel_shape", std::vector{1, 1}).size() != 2) { - LOGS_DEFAULT(VERBOSE) << "Only pooling 2d is supported"; - return false; - } - - if (helper.Get("ceil_mode", 0) == 1) { - LOGS_DEFAULT(VERBOSE) << "ceil_mode == 1 is not supported for pooling"; - return false; - } - - if (helper.Get("dilations", std::vector{1, 1}) != - std::vector{1, 1}) { - LOGS_DEFAULT(VERBOSE) << "Dilations of pooling is not supported"; - return false; - } - - if (node.output_size() != 1) { - LOGS_DEFAULT(VERBOSE) << "Argmax in maxpooling is not supported"; - return false; - } - } else if (op == "GlobalAveragePool" || op == "GlobalMaxPool") { - const auto input_shape = GetShape(model_builder.GetOnnxModel(), node.input(0)); - if (input_shape.size() > 4) { - LOGS_DEFAULT(VERBOSE) - << "GlobalAveragePool/GlobalMaxPool Only rank-4 tensor is supported in " - << node.input(0) << ", actual dim count " << input_shape.size(); - return false; - } - } - - return true; -} - void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); auto input = node.InputDefs()[0]->Name(); bool use_nchw = model_builder.UseNCHW(); @@ -1457,110 +880,23 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - - NodeAttrHelper helper(node); - - auto input = node.input(0); - bool use_nchw = model_builder.UseNCHW(); - bool input_is_nhwc = model_builder.IsOperandNHWC(input); - bool output_is_nhwc = false; - if (use_nchw) { - ORT_ENFORCE(!input_is_nhwc, "model_builder.UseNCHW() but input is NHWC"); - } else { - output_is_nhwc = true; - if (!input_is_nhwc) { - const auto& nchw_input = node.input(0); - if (!model_builder.GetNHWCOperand(nchw_input, input)) { - input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); - TransposeNCHWToNHWC(model_builder, nchw_input, input); - } - } - } - - const auto& output = node.output(0); - const auto& op = node.op_type(); - - int32_t op_type; - if (op == "AveragePool" || op == "GlobalAveragePool") - op_type = ANEURALNETWORKS_AVERAGE_POOL_2D; - else // (op == "MaxPool" || op == "GlobalMaxPool") - op_type = ANEURALNETWORKS_MAX_POOL_2D; - - vector onnx_pads, onnx_strides, kernel_shape; - if (op == "AveragePool" || op == "MaxPool") { - kernel_shape = helper.Get("kernel_shape", vector{0, 0}); - onnx_strides = helper.Get("strides", vector{1, 1}); - onnx_pads = helper.Get("pads", vector{0, 0, 0, 0}); - } else { // (op == "GlobalAveragePool" || op == "GlobalMaxPool") - onnx_strides = vector{1, 1}; - onnx_pads = vector{0, 0, 0, 0}; - if (model_builder.UseNCHW()) - kernel_shape = vector{static_cast(shaper[input][2]), - static_cast(shaper[input][3])}; - else - kernel_shape = vector{static_cast(shaper[input][1]), - static_cast(shaper[input][2])}; - } - - int32_t fuse_code = model_builder.FindActivation(output); - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[1])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[3])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[0])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[2])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[1])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[0])); - input_indices.push_back(model_builder.AddOperandFromScalar(kernel_shape[1])); - input_indices.push_back(model_builder.AddOperandFromScalar(kernel_shape[0])); - input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); - input_indices.push_back(model_builder.AddOperandFromScalar(use_nchw)); - - shaper.Pool(input, - onnx_pads, onnx_strides, kernel_shape, - use_nchw, - output); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); -} - #pragma endregion op_pool #pragma region op_conv class ConvOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; -void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - // skip the weight for conv as we need to transpose - model_builder.AddInitializerToSkip(node.input(1)); -} - void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) { // skip the weight for conv as we need to transpose @@ -1569,34 +905,6 @@ void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { - GraphNodeAttrHelper helper(node); - if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { - LOGS_DEFAULT(VERBOSE) << "SAME_LOWER auto_pad is not supported"; - return false; - } - - const auto group = helper.Get("group", 1); - const auto weight_name = node.InputDefs()[1]->Name(); - if (Contains(model_builder.GetInitializerTensors(), weight_name)) { - const auto& tensor = model_builder.GetInitializerTensors().at(weight_name); - if (tensor.dims().size() != 4) { - LOGS_DEFAULT(VERBOSE) << "Only conv 2d is supported."; - return false; - } - if (group != 1 && tensor.dims()[1] != 1) { - LOGS_DEFAULT(VERBOSE) << "group != 1 is not supported"; - return false; - } - } else { - LOGS_DEFAULT(VERBOSE) << "The weight of convolution must be known"; - return false; - } - - return true; -} - -bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { NodeAttrHelper helper(node); if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { LOGS_DEFAULT(VERBOSE) << "SAME_LOWER auto_pad is not supported"; @@ -1604,7 +912,7 @@ bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, } const auto group = helper.Get("group", 1); - const auto weight_name = node.input(1); + const auto weight_name = node.InputDefs()[1]->Name(); if (Contains(model_builder.GetInitializerTensors(), weight_name)) { const auto& tensor = model_builder.GetInitializerTensors().at(weight_name); if (tensor.dims().size() != 4) { @@ -1629,7 +937,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); // onnx strides are in the order height, width // while nnapi strides are in the order width, height @@ -1744,136 +1052,12 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, model_builder.AddOperation(operationCode, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - const auto& initializers(model_builder.GetInitializerTensors()); - NodeAttrHelper helper(node); - - // onnx strides are in the order height, width - // while nnapi strides are in the order width, height - const auto onnx_strides = helper.Get("strides", vector{1, 1}); - - // onnx pads are in the order top, left, bottom, right - // while nnapi pads is in the order left, right, top, bottom - const auto onnx_pads = helper.Get("pads", vector{0, 0, 0, 0}); - - // onnx dilations is in the order height, width - // while nnapi dilations are in the order width, height - const auto onnx_dilations = helper.Get("dilations", vector{1, 1}); - const auto group = helper.Get("group", 1); - - auto input = node.input(0); - bool use_nchw = model_builder.UseNCHW(); - bool input_is_nhwc = model_builder.IsOperandNHWC(input); - bool output_is_nhwc = false; - if (use_nchw) { - ORT_ENFORCE(!input_is_nhwc, "model_builder.UseNCHW() but input is NHWC"); - } else { - output_is_nhwc = true; - if (!input_is_nhwc) { - const auto& nchw_input = node.input(0); - if (!model_builder.GetNHWCOperand(nchw_input, input)) { - input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); - TransposeNCHWToNHWC(model_builder, nchw_input, input); - } - } - } - - const auto& weight = node.input(1); - const auto& output = node.output(0); - - bool conv2d = (group == 1); - const auto& weight_tensor = initializers.at(weight); - bool depthwise_conv2d = (weight_tensor.dims()[1] == 1); - - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); - - if (conv2d) { - input_indices.push_back(AddInitializerInNewLayout( - model_builder, weight, L_0231)); - } else { // depthwise_conv2d - input_indices.push_back(AddInitializerInNewLayout( - model_builder, weight, L_1230)); - } - - bool hasBias = (node.input_size() >= 3); - std::string bias = hasBias ? node.input(2) : weight + "_bias"; - - uint32_t bias_idx_val; - if (hasBias) { - bias_idx_val = operand_indices.at(bias); - } else { - const auto weight_dimen = shaper[weight]; - ModelBuilder::Shape bias_dimen; - if (conv2d) - bias_dimen = {weight_dimen[0]}; - else - bias_dimen = {weight_dimen[3]}; - - const auto& weight_type = operand_types.at(weight).type; - if (weight_type == Type::TENSOR_FLOAT32) { - vector buffer(bias_dimen[0]); - for (uint32_t i = 0; i < buffer.size(); i++) { - buffer[i] = 0.f; - } - OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); - bias_idx_val = model_builder.AddOperandFromPersistMemoryBuffer( - bias, buffer.data(), operandType); - } else { - ORT_THROW("Unknown weight type " + TypeToStr(weight_type)); - } - } - - input_indices.push_back(bias_idx_val); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[1])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[3])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[0])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[2])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[1])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_strides[0])); - if (!conv2d && depthwise_conv2d) { - int32_t depthwiseMultiplier = shaper[weight][3] / group; - input_indices.push_back(model_builder.AddOperandFromScalar(depthwiseMultiplier)); - } - int32_t fuse_code = model_builder.FindActivation(output); - input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); - // TODO support API 28 - input_indices.push_back(model_builder.AddOperandFromScalar(use_nchw)); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_dilations[1])); - input_indices.push_back(model_builder.AddOperandFromScalar(onnx_dilations[0])); - - int32_t operationCode; - if (conv2d) { - operationCode = ANEURALNETWORKS_CONV_2D; - shaper.Conv(input, weight, - onnx_pads, onnx_strides, onnx_dilations, - use_nchw, - output); - } else { // depthwise_conv2d - operationCode = ANEURALNETWORKS_DEPTHWISE_CONV_2D; - shaper.DepthwiseConv(input, weight, - onnx_pads, onnx_strides, onnx_dilations, - use_nchw, - output); - } - - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - model_builder.AddOperation(operationCode, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); -} - #pragma endregion op_conv #pragma region op_cast class CastOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; @@ -1882,33 +1066,12 @@ class CastOpBuilder : public BaseOpBuilder { return 29; } - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { - return 29; - } - - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const onnxruntime::Node& node) { - GraphNodeAttrHelper helper(node); - auto to = helper.Get("to", 0); - if (to != ONNX_NAMESPACE::TensorProto::FLOAT && - to != ONNX_NAMESPACE::TensorProto::INT32) { - LOGS_DEFAULT(VERBOSE) << "[Cast] Only support cast to int32 or float"; - return false; - } - - return true; -} - -bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) { NodeAttrHelper helper(node); auto to = helper.Get("to", 0); if (to != ONNX_NAMESPACE::TensorProto::FLOAT && @@ -1924,7 +1087,7 @@ void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); const auto& input = node.InputDefs()[0]->Name(); const auto& output = node.OutputDefs()[0]->Name(); @@ -1952,47 +1115,12 @@ void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, {output_operand_type}, {output_is_nhwc}); } -void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - NodeAttrHelper helper(node); - - const auto& input = node.input(0); - const auto& output = node.output(0); - bool output_is_nhwc = model_builder.IsOperandNHWC(input); - - auto to = helper.Get("to", 0); - Type type; - switch (to) { - case ONNX_NAMESPACE::TensorProto::FLOAT: - type = Type::TENSOR_FLOAT32; - break; - case ONNX_NAMESPACE::TensorProto::INT32: - type = Type::TENSOR_INT32; - break; - default: - ORT_THROW("Invalid cast to type: " + - std::to_string(to)); - } - - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); - shaper.Identity(input, output); - const OperandType output_operand_type(type, shaper[output]); - model_builder.AddOperation(ANEURALNETWORKS_CAST, input_indices, {output}, - {output_operand_type}, {output_is_nhwc}); -} - #pragma endregion #pragma region op_softmax class SoftMaxOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; @@ -2001,89 +1129,39 @@ class SoftMaxOpBuilder : public BaseOpBuilder { return 29; } - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& /* node */) const override { - return 29; - } - - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) { - const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); - if (input_size != 2 && input_size != 4) { - LOGS_DEFAULT(VERBOSE) << "SoftMax only support 2d/4d shape, input is " - << input_size << "d shape"; - return false; - } - return true; -} - -bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto input_size = GetShape(model_builder.GetOnnxModel(), node.input(0)).size(); - if (input_size != 2 && input_size != 4) { - LOGS_DEFAULT(VERBOSE) << "SoftMax only support 2d/4d shape, input is " - << input_size << "d shape"; - return false; - } - return true; -} - -void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - GraphNodeAttrHelper helper(node); - - auto input = node.InputDefs()[0]->Name(); - if (model_builder.IsOperandNHWC(input)) { - // We want to transpose nhwc operand back to nchw before softmax - const auto& nhwc_input = node.InputDefs()[0]->Name(); - if (!model_builder.GetNCHWOperand(nhwc_input, input)) { - input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); - TransposeNHWCToNCHW(model_builder, nhwc_input, input); - } - } - - const auto& output = node.OutputDefs()[0]->Name(); - float beta = 1.f; - int32_t axis = helper.Get("axis", 1); - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); - input_indices.push_back(model_builder.AddOperandFromScalar(beta)); - input_indices.push_back(model_builder.AddOperandFromScalar(axis)); - - shaper.Identity(input, output); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - model_builder.AddOperation(ANEURALNETWORKS_SOFTMAX, input_indices, {output}, - {output_operand_type}, {false}); + const onnxruntime::Node& node) { + const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + if (input_size != 2 && input_size != 4) { + LOGS_DEFAULT(VERBOSE) << "SoftMax only support 2d/4d shape, input is " + << input_size << "d shape"; + return false; + } + return true; } void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { + const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); NodeAttrHelper helper(node); - auto input = node.input(0); + auto input = node.InputDefs()[0]->Name(); if (model_builder.IsOperandNHWC(input)) { // We want to transpose nhwc operand back to nchw before softmax - const auto& nhwc_input = node.input(0); + const auto& nhwc_input = node.InputDefs()[0]->Name(); if (!model_builder.GetNCHWOperand(nhwc_input, input)) { input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); TransposeNHWCToNCHW(model_builder, nhwc_input, input); } } - const auto& output = node.output(0); + const auto& output = node.OutputDefs()[0]->Name(); float beta = 1.f; int32_t axis = helper.Get("axis", 1); std::vector input_indices; @@ -2103,9 +1181,6 @@ void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class IdentityOpBuilder : public BaseOpBuilder { private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -2130,48 +1205,19 @@ void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); } -void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - // Identity is not really going to do anything - // Just register the dimension and type, with same index and new name - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - - const auto& input = node.input(0); - const auto& output = node.output(0); - bool output_is_nhwc = model_builder.IsOperandNHWC(input); - - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); // input - - shaper.Identity(input, output); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type, output_is_nhwc); -} - #pragma endregion #pragma region op_gemm class GemmOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -2200,7 +1246,7 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, // Only support // 1. A*B'+C // 2. A*B+C and B is an initializer - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); const auto transA = helper.Get("transA", 0); const auto transB = helper.Get("transB", 0); const auto alpha = helper.Get("alpha", 1.0f); @@ -2234,85 +1280,13 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -bool GemmOpBuilder::IsOpSupportedImpl( - ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); - const auto& initializers(model_builder.GetInitializerTensors()); - - if (GetShape(model_builder.GetOnnxModel(), node.input(0)).size() != 2) { - LOGS_DEFAULT(VERBOSE) << "A must be 2D"; - return false; - } - - if (GetShape(model_builder.GetOnnxModel(), node.input(0)).size() != 2) { - LOGS_DEFAULT(VERBOSE) << "B must be 2D"; - return false; - } - - if (op == "MatMul") { // Only support A*B B is an initializer - if (!Contains(initializers, node.input(1))) { - LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known"; - return false; - } - } else if (op == "Gemm") { - // Only support - // 1. A*B'+C - // 2. A*B+C and B is an initializer - NodeAttrHelper helper(node); - const auto transA = helper.Get("transA", 0); - const auto transB = helper.Get("transB", 0); - const auto alpha = helper.Get("alpha", 1.0f); - const auto beta = helper.Get("beta", 1.0f); - - if (!(transA == 0 && alpha == 1.f && beta == 1.f)) { - LOGS_DEFAULT(VERBOSE) << "Only transA == 0, alpha == 1.0 " - << "and beta == 1.0 is supported."; - return false; - } - - if (transB == 0 && !Contains(initializers, node.input(1))) { - LOGS_DEFAULT(VERBOSE) << "B of Gemm must be known if transB != 1"; - return false; - } - - if (node.input_size() == 3) { - const auto b_shape = GetShape(model_builder.GetOnnxModel(), node.input(1)); - const auto c_shape = GetShape(model_builder.GetOnnxModel(), node.input(2)); - if (c_shape.size() != 1 || - c_shape[0] != (transB == 0 ? b_shape[1] : b_shape[0])) { - LOGS_DEFAULT(VERBOSE) << "C of Gemm must be a vector of b_shape[0]" - << " b_shape: " << Shape2String(b_shape) - << " c_shape: " << Shape2String(c_shape); - - return false; - } - } - } - - return true; -} - -void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); - if (op == "MatMul") { - model_builder.AddInitializerToSkip(node.input(1)); - } else if (op == "Gemm") { - NodeAttrHelper helper(node); - const auto transB = helper.Get("transB", 0); - if (transB == 0) - model_builder.AddInitializerToSkip(node.input(1)); - } -} - void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) { const auto& op = node.OpType(); if (op == "MatMul") { model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } else if (op == "Gemm") { - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); const auto transB = helper.Get("transB", 0); if (transB == 0) model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); @@ -2325,7 +1299,7 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); const auto& input1 = node.InputDefs()[0]->Name(); const auto& input2 = node.InputDefs()[1]->Name(); @@ -2372,59 +1346,6 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, {output_operand_type}, {false}); } -void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - const auto& op = node.op_type(); - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - NodeAttrHelper helper(node); - - const auto& input1 = node.input(0); - const auto& input2 = node.input(1); - const auto& output = node.output(0); - const auto transB = helper.Get("transB", 0); - - uint32_t input_2_idx; - if (transB == 0) { - input_2_idx = AddInitializerTransposed(model_builder, input2); - } else { - input_2_idx = operand_indices.at(input2); - } - - uint32_t bias_idx; - if (node.input_size() == 2) { - std::string bias = node.name() + op + "_bias"; - const auto& B_type = operand_types.at(input2).type; - ModelBuilder::Shape bias_dimen = {shaper[input2][0]}; - if (B_type == Type::TENSOR_FLOAT32) { - float buffer[bias_dimen[0]]; - for (uint32_t i = 0; i < bias_dimen[0]; i++) { - buffer[i] = 0.f; - } - OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); - bias_idx = model_builder.AddOperandFromPersistMemoryBuffer( - bias, &buffer[0], operandType); - } else { - ORT_THROW("Unknown weight type " + TypeToStr(B_type)); - } - } else { - bias_idx = operand_indices.at(node.input(2)); - } - - std::vector input_indices; - input_indices.push_back(operand_indices.at(input1)); // A - input_indices.push_back(input_2_idx); // B - input_indices.push_back(bias_idx); // C - int32_t fuse_code = model_builder.FindActivation(output); - input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); - - shaper.FC(input1, input2, output); - const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); - model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices, {output}, - {output_operand_type}, {false}); -} - #pragma endregion #pragma region op_unary @@ -2434,12 +1355,6 @@ class UnaryOpBuilder : public BaseOpBuilder { int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const onnxruntime::Node& node) const override; - int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) const override; - - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -2459,21 +1374,6 @@ int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, return 27; } -int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const ONNX_NAMESPACE::NodeProto& node) const { - const auto& op(node.op_type()); - if (op == "Abs" || - op == "Exp" || - op == "Neg" || - op == "Sin" || - op == "Sqrt" || - op == "Log") { - return 29; - } - - return 27; -} - void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); @@ -2515,62 +1415,15 @@ void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - const auto& op(node.op_type()); - - const auto& input = node.input(0); - const auto& output = node.output(0); - bool output_is_nhwc = model_builder.IsOperandNHWC(input); - - shaper.Identity(input, output); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - - int32_t op_code; - if (op == "Abs") - op_code = ANEURALNETWORKS_ABS; - else if (op == "Exp") - op_code = ANEURALNETWORKS_EXP; - else if (op == "Floor") - op_code = ANEURALNETWORKS_FLOOR; - else if (op == "Log") - op_code = ANEURALNETWORKS_LOG; - else if (op == "Sigmoid") - op_code = ANEURALNETWORKS_LOGISTIC; - else if (op == "Neg") - op_code = ANEURALNETWORKS_NEG; - else if (op == "Sin") - op_code = ANEURALNETWORKS_SIN; - else if (op == "Sqrt") - op_code = ANEURALNETWORKS_SQRT; - else if (op == "Tanh") - op_code = ANEURALNETWORKS_TANH; - else { - ORT_THROW("UnaryOpBuilder, unknown op: " + op); - } - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); - model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); -} - #pragma endregion #pragma region op_concat class ConcatOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) override; }; @@ -2587,22 +1440,12 @@ bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - if (GetShape(model_builder.GetOnnxModel(), node.input(0)).size() > 4) { - LOGS_DEFAULT(VERBOSE) << "Concat supports at most 4D shape"; - return false; - } - - return true; -} - void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - GraphNodeAttrHelper helper(node); + NodeAttrHelper helper(node); std::vector input_indices; const auto& input0 = node.InputDefs()[0]->Name(); @@ -2668,73 +1511,6 @@ void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, {output_operand_type}, {output_is_nhwc}); } -void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) { - auto& shaper(model_builder.GetShaper()); - const auto& operand_indices(model_builder.GetOperandIndices()); - const auto& operand_types(model_builder.GetOperandTypes()); - NodeAttrHelper helper(node); - - std::vector input_indices; - const auto& input0 = node.input(0); - bool all_input_have_same_layout = true; - bool output_is_nhwc = false; - - // First we want to see if all the input are smae layout - for (int i = 0; i < node.input_size() - 1; i++) { - all_input_have_same_layout = - all_input_have_same_layout && - model_builder.IsOperandNHWC(node.input(i)) == model_builder.IsOperandNHWC(node.input(i + 1)); - } - - std::vector inputs; - inputs.reserve(node.input_size()); - if (all_input_have_same_layout) { - // if all the inputs are of same layout, output will be the same layout - if (model_builder.IsOperandNHWC(input0)) { - output_is_nhwc = true; - } - - for (const auto& input : node.input()) { - input_indices.push_back(operand_indices.at(input)); - inputs.push_back(input); - } - } else { - // if all the inputs are not same layout, - // will need transpos those nhwc tensors back to nchw - for (auto input : node.input()) { - if (model_builder.IsOperandNHWC(input)) { - std::string nhwc_input = input; - input = model_builder.GetUniqueName(input + "_nhwc_to_nchw"); - TransposeNHWCToNCHW(model_builder, nhwc_input, input); - } - input_indices.push_back(operand_indices.at(input)); - inputs.push_back(input); - } - } - - int32_t axis = helper.Get("axis", 1); - int rank = shaper[input0].size(); - if (axis < 0) { // NNAPI does not support negative axis - axis = rank + axis; - } - - if (output_is_nhwc) { - ORT_ENFORCE(rank == 4, "nhwc is only on 4d shape, input " + input0 + - " has rank: " + std::to_string(rank)); - // we are using nhwc here, but the axis is in nwhw, need to transpose axis from nchw to nhwc - const uint32_t axis_nchw_to_nhwc[4]{0, 3, 1, 2}; - axis = axis_nchw_to_nhwc[axis]; - } - input_indices.push_back(model_builder.AddOperandFromScalar(axis)); - - const auto& output = node.output(0); - shaper.Concat(inputs, axis, output); - const OperandType output_operand_type(operand_types.at(input0).type, shaper[output]); - model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, input_indices, {output}, - {output_operand_type}, {output_is_nhwc}); -} - #pragma endregion #pragma region CreateOpBuilders diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index f8d3a46e45f3c..d9785a43468ed 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -13,26 +13,14 @@ class IOpBuilder { virtual ~IOpBuilder() = default; // Check if an operator is supported - virtual bool IsOpSupported(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) = 0; - virtual bool IsOpSupported(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; - // Check if the initializers of this operator need preprocess - // which will not be copied - virtual void AddInitializersToSkip(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) = 0; - // Check if the initializers of this operator need preprocess // which will not be copied virtual void AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; - // Add the operator to NNAPI model - virtual void AddToModelBuilder(ModelBuilder& model_builder, - const ONNX_NAMESPACE::NodeProto& node) = 0; - // Add the operator to NNAPI model virtual void AddToModelBuilder(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 20ebee5b690e6..c043f5d666a2e 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -53,41 +53,14 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view } } - // TODO, switch to use graph instead of model - // This method is based on that of TRT EP - // Construct modelproto from graph - onnxruntime::Model model(graph_view.Name(), true, ModelMetaData(), - PathString(), - IOnnxRuntimeOpSchemaRegistryList(), - graph_view.DomainToVersionMap(), - std::vector(), - *GetLogger()); std::unordered_set all_node_inputs; - onnxruntime::Graph& graph_build = model.MainGraph(); for (const auto& node : graph_view.Nodes()) { - std::vector inputs, outputs; for (auto* input : node.InputDefs()) { - auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto()); - inputs.push_back(&n_input); all_node_inputs.insert(input->Name()); } - for (auto* output : node.OutputDefs()) { - auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto()); - outputs.push_back(&n_output); - } - graph_build.AddNode(node.Name(), node.OpType(), node.Description(), inputs, outputs, &node.GetAttributes(), node.Domain()); - } - //Add initializer to graph - const auto& init_tensors = graph_view.GetAllInitializedTensors(); - for (const auto& tensor : init_tensors) { - graph_build.AddInitializedTensor(*(tensor.second)); } - ORT_ENFORCE(graph_build.Resolve().IsOK()); - ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); - model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); - - nnapi::ModelBuilder builder(model_proto, graph_view); + nnapi::ModelBuilder builder(graph_view); const auto supported_nodes_vector = builder.GetSupportedNodes(); // Find inputs, initializers and outputs for each supported subgraph @@ -222,13 +195,10 @@ common::Status NnapiExecutionProvider::Compile(const std::vector(), *GetLogger()); - ONNX_NAMESPACE::ModelProto model_proto = model.ToProto(); - *(model_proto.mutable_graph()) = graph_body.ToGraphProto(); - model_proto.set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); { onnxruntime::GraphViewer graph_viewer(graph_body); - nnapi::ModelBuilder builder(model_proto, graph_viewer); + nnapi::ModelBuilder builder(graph_viewer); builder.SetUseNCHW(false); builder.SetUseFp16(false); std::unique_ptr nnapi_model = builder.Compile(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h index 9cf99f5daef28..426a10ea7fe89 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h @@ -17,8 +17,6 @@ #ifndef ANDROID_ML_NN_RUNTIME_NEURAL_NETWORKS_WRAPPER_H #define ANDROID_ML_NN_RUNTIME_NEURAL_NETWORKS_WRAPPER_H #include "nnapi_implementation.h" -#include -#include #include #include From 6fdfad503030b8d8f3bda894c5293f8feb34e204 Mon Sep 17 00:00:00 2001 From: gwang Date: Tue, 30 Jun 2020 23:04:04 -0700 Subject: [PATCH 05/11] Clear some redundant code --- .../nnapi/nnapi_builtin/builders/model_builder.cc | 14 -------------- .../nnapi/nnapi_builtin/builders/model_builder.h | 5 +---- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 20670068fbbb0..a94fa359d1ef4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include -#include #include #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" @@ -16,12 +15,6 @@ namespace nnapi { using namespace android::nn::wrapper; using std::vector; -const float* GetTensorFloatDataA(const ONNX_NAMESPACE::TensorProto& tensor) { - return tensor.float_data().empty() - ? reinterpret_cast(tensor.raw_data().data()) - : tensor.float_data().data(); -} - ModelBuilder::ModelBuilder(const onnxruntime::GraphViewer& graph_view) : nnapi_(NnApiImplementation()), graph_view_(graph_view) { GetAllInitializers(); @@ -525,13 +518,6 @@ int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, return fuse_code; } -IOpBuilder* ModelBuilder::GetOpBuilder(const ONNX_NAMESPACE::NodeProto& node) { - if (!Contains(op_builders_, node.op_type())) - return nullptr; - - return op_builders_[node.op_type()].get(); -} - IOpBuilder* ModelBuilder::GetOpBuilder(const onnxruntime::Node& node) { if (!Contains(op_builders_, node.OpType())) return nullptr; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index e8ff55f9d8023..f5325f87c8bb4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -91,8 +91,7 @@ class ModelBuilder { const std::unordered_set& GetFusedActivations() const { return fused_activations_; } - const std::unordered_map& + const std::unordered_map& GetInitializerTensors() const { return initializers_; } const onnxruntime::Graph& GetOnnxGraph() const { return graph_view_.GetGraph(); } @@ -151,7 +150,6 @@ class ModelBuilder { uint32_t next_index_ = 0; - bool IsNodeSupported(const ONNX_NAMESPACE::NodeProto& node); bool IsNodeSupported(const onnxruntime::Node& node); // Convert the onnx model to ANeuralNetworksModel @@ -174,7 +172,6 @@ class ModelBuilder { const android::nn::wrapper::OperandType& operand_type, bool is_nhwc); - IOpBuilder* GetOpBuilder(const ONNX_NAMESPACE::NodeProto& node); IOpBuilder* GetOpBuilder(const onnxruntime::Node& node); }; From 18cc272256cae996a2c766213efa14e00972ab25 Mon Sep 17 00:00:00 2001 From: gwang Date: Wed, 1 Jul 2020 15:26:52 -0700 Subject: [PATCH 06/11] merge master and some minor style changes --- .../nnapi/nnapi_builtin/builders/helper.cc | 29 ++++++++++++++++++- .../nnapi/nnapi_builtin/builders/helper.h | 28 +----------------- .../nnapi_builtin/builders/op_builder.cc | 22 ++++---------- .../nnapi/nnapi_builtin/builders/op_builder.h | 17 ++++------- 4 files changed, 41 insertions(+), 55 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index f0c9f69126e40..e12b480501d88 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -3,7 +3,6 @@ // #include -#include #include #include #include @@ -13,6 +12,34 @@ using std::string; using std::vector; +std::string GetErrorCause(int error_code) { + switch (error_code) { + case ANEURALNETWORKS_NO_ERROR: + return "ANEURALNETWORKS_NO_ERROR"; + case ANEURALNETWORKS_OUT_OF_MEMORY: + return "ANEURALNETWORKS_OUT_OF_MEMORY"; + case ANEURALNETWORKS_INCOMPLETE: + return "ANEURALNETWORKS_INCOMPLETE"; + case ANEURALNETWORKS_UNEXPECTED_NULL: + return "ANEURALNETWORKS_UNEXPECTED_NULL"; + case ANEURALNETWORKS_BAD_DATA: + return "ANEURALNETWORKS_BAD_DATA"; + case ANEURALNETWORKS_OP_FAILED: + return "ANEURALNETWORKS_OP_FAILED"; + case ANEURALNETWORKS_BAD_STATE: + return "ANEURALNETWORKS_BAD_STATE"; + case ANEURALNETWORKS_UNMAPPABLE: + return "ANEURALNETWORKS_UNMAPPABLE"; + case ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE: + return "ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE"; + case ANEURALNETWORKS_UNAVAILABLE_DEVICE: + return "ANEURALNETWORKS_UNAVAILABLE_DEVICE"; + + default: + return "Unknown error code: " + std::to_string(error_code); + } +} + NodeAttrHelper::NodeAttrHelper(const onnxruntime::Node& node) : node_attributes_(node.GetAttributes()) {} diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index 6d8662beefa3f..605f5180aa75b 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -34,33 +34,7 @@ inline bool Contains(const Map& map, const Key& key) { return map.find(key) != map.end(); } -inline std::string GetErrorCause(int error_code) { - switch (error_code) { - case ANEURALNETWORKS_NO_ERROR: - return "ANEURALNETWORKS_NO_ERROR"; - case ANEURALNETWORKS_OUT_OF_MEMORY: - return "ANEURALNETWORKS_OUT_OF_MEMORY"; - case ANEURALNETWORKS_INCOMPLETE: - return "ANEURALNETWORKS_INCOMPLETE"; - case ANEURALNETWORKS_UNEXPECTED_NULL: - return "ANEURALNETWORKS_UNEXPECTED_NULL"; - case ANEURALNETWORKS_BAD_DATA: - return "ANEURALNETWORKS_BAD_DATA"; - case ANEURALNETWORKS_OP_FAILED: - return "ANEURALNETWORKS_OP_FAILED"; - case ANEURALNETWORKS_BAD_STATE: - return "ANEURALNETWORKS_BAD_STATE"; - case ANEURALNETWORKS_UNMAPPABLE: - return "ANEURALNETWORKS_UNMAPPABLE"; - case ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE: - return "ANEURALNETWORKS_OUTPUT_INSUFFICIENT_SIZE"; - case ANEURALNETWORKS_UNAVAILABLE_DEVICE: - return "ANEURALNETWORKS_UNAVAILABLE_DEVICE"; - - default: - return "Unknown error code: " + std::to_string(error_code); - } -} +std::string GetErrorCause(int error_code); /** * Wrapping onnxruntime::Node for retrieving attribute values diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index d9edd6cdc37ed..3cf28e2070d70 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -263,24 +263,19 @@ class BaseOpBuilder : public IOpBuilder { virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, const onnxruntime::Node& /* node */) override {} - bool IsOpSupported(ModelBuilder& model_builder, - const onnxruntime::Node& node) override final; + bool IsOpSupported(ModelBuilder& model_builder, const onnxruntime::Node& node) override final; - void AddToModelBuilder(ModelBuilder& model_builder, - const onnxruntime::Node& node) override final; + void AddToModelBuilder(ModelBuilder& model_builder, const onnxruntime::Node& node) override final; protected: - virtual bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node); + virtual bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node); - virtual int32_t GetMinSupportedSdkVer( - ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) const { return 27; } + virtual int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, + const onnxruntime::Node& /* node */) const { return 27; } virtual bool HasSupportedInputs(const onnxruntime::Node& node); - virtual void AddToModelBuilderImpl( - ModelBuilder& model_builder, const onnxruntime::Node& node); + virtual void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; }; bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, @@ -334,11 +329,6 @@ void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, << "] type: [" << node.OpType() << "] was added"; } -void BaseOpBuilder::AddToModelBuilderImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) { - ORT_NOT_IMPLEMENTED("Unsupported operator " + node.OpType()); -} - #pragma endregion op_base #pragma region op_binary diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index d9785a43468ed..8070e1c968b1a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -13,27 +13,22 @@ class IOpBuilder { virtual ~IOpBuilder() = default; // Check if an operator is supported - virtual bool IsOpSupported(ModelBuilder& model_builder, - const onnxruntime::Node& node) = 0; + virtual bool IsOpSupported(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; // Check if the initializers of this operator need preprocess // which will not be copied - virtual void AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) = 0; + virtual void AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; // Add the operator to NNAPI model - virtual void AddToModelBuilder(ModelBuilder& model_builder, - const onnxruntime::Node& node) = 0; + virtual void AddToModelBuilder(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; }; // Generate a lookup table with IOpBuilder delegates // for different onnx operators -std::unordered_map> -CreateOpBuilders(); +std::unordered_map> CreateOpBuilders(); -void TransposeNHWCToNCHW(ModelBuilder& model_builder, - const std::string& input, - const std::string& output); +// Transpose the NHWCinput to NCHW output +void TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output); } // namespace nnapi } // namespace onnxruntime From 44f3b3ef398d9b01ddc7755ba7d9b4797d17cc92 Mon Sep 17 00:00:00 2001 From: gwang Date: Wed, 1 Jul 2020 16:54:39 -0700 Subject: [PATCH 07/11] move check if an initializer is external to individual op instead the whole graph --- .../nnapi_builtin/builders/op_builder.cc | 26 +++++++++++++++++++ .../nnapi_builtin/nnapi_execution_provider.cc | 10 ------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 3cf28e2070d70..a8b9da219c3c8 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -276,6 +276,8 @@ class BaseOpBuilder : public IOpBuilder { virtual bool HasSupportedInputs(const onnxruntime::Node& node); virtual void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; + + bool HasExternalInitializer(ModelBuilder& model_builder, const onnxruntime::Node& node); }; bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, @@ -294,6 +296,10 @@ bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, if (!HasSupportedInputs(node)) return false; + // We do not support external initializers for now + if (HasExternalInitializer(model_builder, node)) + return false; + return IsOpSupportedImpl(model_builder, node); } @@ -329,6 +335,26 @@ void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, << "] type: [" << node.OpType() << "] was added"; } +bool BaseOpBuilder::HasExternalInitializer(ModelBuilder& model_builder, + const onnxruntime::Node& node) { + const auto& initializers(model_builder.GetOnnxGraph().GetAllInitializedTensors()); + for (const auto* node_arg : node.InputDefs()) { + const auto& input_name(node_arg->Name()); + if (!Contains(initializers, input_name)) + continue; + + const auto* tensor = initializers.at(input_name); + if (tensor->has_data_location() && + tensor->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { + LOGS_DEFAULT(VERBOSE) << "Initializer [" << input_name + << "] with external data location are not currently supported"; + return true; + } + } + + return false; +} + #pragma endregion op_base #pragma region op_binary diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index c043f5d666a2e..f284dd1851535 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -43,16 +43,6 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const std::vector& /*kernel_registries*/) const { std::vector> result; - // Need access to model_path_ - for (const auto& tensor : graph_view.GetAllInitializedTensors()) { - if (tensor.second->has_data_location() && - tensor.second->data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) { - LOGS_DEFAULT(WARNING) << "NNAPI: Initializers with external data" - " location are not currently supported"; - return result; - } - } - std::unordered_set all_node_inputs; for (const auto& node : graph_view.Nodes()) { for (auto* input : node.InputDefs()) { From 64b5817e72d095c4ba3776d28e996321e047d5bd Mon Sep 17 00:00:00 2001 From: gwang Date: Wed, 1 Jul 2020 21:33:25 -0700 Subject: [PATCH 08/11] Addressed comments --- .../nnapi_builtin/builders/model_builder.cc | 32 ++--- .../nnapi_builtin/builders/model_builder.h | 2 +- .../nnapi_builtin/builders/op_builder.cc | 135 +++++++++++++----- 3 files changed, 109 insertions(+), 60 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index a94fa359d1ef4..f453bbdcb5407 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -40,7 +40,6 @@ bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, if (supported_node_vec.size() == 1) { const auto& node_indices = graph_view.GetNodesInTopologicalOrder(); const auto* node(graph_view.GetNode(node_indices[supported_node_vec[0]])); - ORT_ENFORCE(nullptr != node, "node should not be null"); const auto& op = node->OpType(); // It is not worth it to perform a single Reshape/Dropout/Identity operator // which is only copying the data in NNAPI @@ -72,7 +71,6 @@ std::vector> ModelBuilder::GetSupportedNodes() { const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); for (size_t i = 0; i < node_indices.size(); i++) { const auto* node(graph_view_.GetNode(node_indices[i])); - ORT_ENFORCE(nullptr != node, "node should not be null"); bool supported = IsNodeSupported(*node); LOGS_DEFAULT(VERBOSE) << "Operator type: [" << node->OpType() << "] index: [" << i @@ -176,7 +174,6 @@ void ModelBuilder::GetTargetDevices() { void ModelBuilder::GetAllInitializers() { for (const auto& pair : graph_view_.GetAllInitializedTensors()) { - ORT_ENFORCE(pair.second != nullptr, "Initializer is null for: " + pair.first); initializers_.emplace(pair.first, *pair.second); } } @@ -185,9 +182,8 @@ void ModelBuilder::PreprocessInitializers() { const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); for (size_t i = 0; i < node_indices.size(); i++) { const auto* node(graph_view_.GetNode(node_indices[i])); - ORT_ENFORCE(nullptr != node, "node should not be null"); - if (auto* opBuilder = GetOpBuilder(*node)) { - opBuilder->AddInitializersToSkip(*this, *node); + if (auto* op_builder = GetOpBuilder(*node)) { + op_builder->AddInitializersToSkip(*this, *node); } } } @@ -263,7 +259,6 @@ void ModelBuilder::RegisterInitializers() { void ModelBuilder::RegisterModelInputs() { for (const auto* node_arg : graph_view_.GetInputs()) { - ORT_ENFORCE(node_arg != nullptr, "input cannot be null"); const auto& input_name = node_arg->Name(); { // input should not be an initializer @@ -312,7 +307,6 @@ void ModelBuilder::RegisterModelInputs() { void ModelBuilder::RegisterModelOutputs() { for (const auto* node_arg : graph_view_.GetOutputs()) { - ORT_ENFORCE(node_arg != nullptr, "input cannot be null"); const auto& output_name = node_arg->Name(); if (!Contains(operands_, output_name)) { @@ -410,12 +404,10 @@ void ModelBuilder::AddOperations() { const auto& node_indices = graph_view_.GetNodesInTopologicalOrder(); for (size_t i = 0; i < node_indices.size(); i++) { const auto* node(graph_view_.GetNode(node_indices[i])); - ORT_ENFORCE(nullptr != node, "node should not be null"); - if (auto* opBuilder = GetOpBuilder(*node)) { - opBuilder->AddToModelBuilder(*this, *node); + if (auto* op_builder = GetOpBuilder(*node)) { + op_builder->AddToModelBuilder(*this, *node); } else { - throw std::invalid_argument( - "Node not supported" + node->Name()); + ORT_THROW("Node [" + node->Name() + "], type [" + node->OpType() + "] is not supported"); } } } @@ -483,21 +475,19 @@ std::unique_ptr ModelBuilder::Compile() { return std::move(nnapi_model_); } -int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, - const NodeArg* output) { - ORT_ENFORCE(output != nullptr, "output cannot be null"); +int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, const NodeArg& output) { int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) { const auto& dst_node = it->GetNode(); const auto* dst_input = dst_node.InputDefs()[it->GetDstArgIndex()]; if (dst_node.OpType() == "Relu") { - if (output == dst_input) { + if (&output == dst_input) { fuse_code = ANEURALNETWORKS_FUSED_RELU; } } else { // if there is any other non-relu node using the output // will add relu separately - if (output == dst_input) + if (&output == dst_input) return ANEURALNETWORKS_FUSED_NONE; } } @@ -505,14 +495,14 @@ int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, // if output is a graph output, will add relu separately if (fuse_code != ANEURALNETWORKS_FUSED_NONE) { for (const auto* graph_output : graph_view_.GetOutputs()) { - if (output == graph_output) + if (&output == graph_output) return ANEURALNETWORKS_FUSED_NONE; } LOGS_DEFAULT(VERBOSE) << "Node [" << node.Name() << "] type [" << node.OpType() - << "], fused the output [" << output->Name() << "]"; + << "], fused the output [" << output.Name() << "]"; - fused_activations_.insert(output->Name()); + fused_activations_.insert(output.Name()); } return fuse_code; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index f5325f87c8bb4..eb21bf82e05b8 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -44,7 +44,7 @@ class ModelBuilder { const std::vector& is_nhwc_vec); // Find if an output has a fuseable activation (Relu) - int32_t FindActivation(const onnxruntime::Node& node, const NodeArg* output); + int32_t FindActivation(const onnxruntime::Node& node, const NodeArg& output); // Add an NNAPI scalar operand uint32_t AddOperandFromScalar(bool value); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index a8b9da219c3c8..9a89f6946e27a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -14,6 +14,7 @@ namespace nnapi { using namespace android::nn::wrapper; using std::vector; +using Shape = Shaper::Shape; #pragma region helpers @@ -42,7 +43,7 @@ void AddTransposeOperator(ModelBuilder& model_builder, std::vector input_indices; input_indices.push_back(operand_indices.at(input)); // input - ModelBuilder::Shape perm_dimen = {SafeInt(perm.size())}; + Shape perm_dimen = {SafeInt(perm.size())}; OperandType perm_operand_type(Type::TENSOR_INT32, perm_dimen); uint32_t perm_idx = model_builder.AddOperandFromPersistMemoryBuffer( perm_name, perm.data(), perm_operand_type); @@ -123,11 +124,9 @@ void AddBinaryOperator(int32_t op_type, model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -bool GetType(const onnxruntime::Node& node, int32_t& type) { +bool GetType(const ONNX_NAMESPACE::TypeProto* type_proto, int32_t& type) { type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; - auto* type_proto = node.InputDefs()[0]->TypeAsProto(); - if (!type_proto || !type_proto->tensor_type().has_elem_type()) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] has no input type"; + if (!type_proto || !type_proto->has_tensor_type() || !type_proto->tensor_type().has_elem_type()) { return false; } @@ -135,16 +134,17 @@ bool GetType(const onnxruntime::Node& node, int32_t& type) { return true; } -Shaper::Shape GetShape(const ONNX_NAMESPACE::TensorShapeProto* shape_proto) { - Shaper::Shape shape; +bool GetShape(const ONNX_NAMESPACE::TensorShapeProto* shape_proto, Shape& shape) { + shape.clear(); + if (!shape_proto) - return shape; + return false; + // NNAPI uses 0 for dynamic dimension, which is the default value for dim.dim_value() for (const auto& dim : shape_proto->dim()) - // NNAPI uses 0 for dynamic dimension, which is the default value for dim.dim_value() shape.push_back(SafeInt(dim.dim_value())); - return shape; + return true; } enum DataLayout { @@ -157,7 +157,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, const std::string& name, DataLayout new_layout) { const auto& tensor = model_builder.GetInitializerTensors().at(name); - ModelBuilder::Shape shape; + Shape shape; for (auto dim : tensor.dims()) shape.push_back(SafeInt(dim)); @@ -175,7 +175,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, auto out_t = shape[0], in_t = shape[1], h_t = shape[2], w_t = shape[3]; - ModelBuilder::Shape dest_shape; + Shape dest_shape; if (new_layout == L_0231) dest_shape = {out_t, h_t, w_t, in_t}; // L_0231 else @@ -221,7 +221,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, uint32_t AddInitializerTransposed(ModelBuilder& model_builder, const std::string& name) { const auto& tensor = model_builder.GetInitializerTensors().at(name); - ModelBuilder::Shape shape; + Shape shape; for (auto dim : tensor.dims()) shape.push_back(SafeInt(dim)); @@ -238,7 +238,7 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, } auto x_t = shape[0], y_t = shape[1]; - ModelBuilder::Shape dest_shape = {y_t, x_t}; + Shape dest_shape = {y_t, x_t}; const OperandType operandType(type, dest_shape); const float* src = GetTensorFloatData(tensor); float* buffer = new float[Product(shape)]; @@ -307,8 +307,10 @@ bool BaseOpBuilder::HasSupportedInputs(const onnxruntime::Node& node) { // We only check the type of input 0 by default // specific op builder can override this int32_t input_type; - if (!GetType(node, input_type)) + if (!GetType(node.InputDefs()[0]->TypeAsProto(), input_type)) { + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] input 0 has no input type"; return false; + } if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() @@ -420,7 +422,7 @@ void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, } } - int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); AddBinaryOperator(op_code, model_builder, input1, input2, fuse_code, output, output_is_nhwc); } @@ -477,7 +479,14 @@ class TransposeOpBuilder : public BaseOpBuilder { bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const onnxruntime::Node& node) { - const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + Shape input_shape; + if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape info"; + return false; + } + + const auto input_size = input_shape.size(); if (input_size > 4 || input_size == 0) { LOGS_DEFAULT(VERBOSE) << "Transpose only supports 1-4d shape, input is " << input_size << "d shape"; @@ -552,7 +561,13 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return false; } - const auto input_shape = GetShape(node.InputDefs()[0]->Shape()); + Shape input_shape; + if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape info"; + return false; + } + if (input_shape.size() > 4 || input_shape.empty()) { LOGS_DEFAULT(VERBOSE) << "Reshape only supports up to 1-4d shape, input is " << input_shape.size() << "d shape"; @@ -600,7 +615,7 @@ void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const int64_t* rawShape = GetTensorInt64Data(shape_tensor); const auto size = SafeInt(shape_tensor.dims()[0]); - ModelBuilder::Shape input_shape = shaper[input]; + Shape input_shape = shaper[input]; std::vector shape(size); for (uint32_t i = 0; i < size; i++) { int32_t dim = SafeInt(rawShape[i]); @@ -608,7 +623,7 @@ void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, shape[i] = dim == 0 ? input_shape[i] : dim; } - ModelBuilder::Shape shape_dimen = {size}; + Shape shape_dimen = {size}; std::string shape_name = model_builder.GetUniqueName(node.Name() + input + "newshape"); OperandType shape_operand_type(Type::TENSOR_INT32, shape_dimen); uint32_t shape_idx = model_builder.AddOperandFromPersistMemoryBuffer(shape_name, shape.data(), shape_operand_type); @@ -716,7 +731,7 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil const auto tensor_a_name = model_builder.GetUniqueName(node.Name() + input + "_imm_a"); const auto tensor_b_name = model_builder.GetUniqueName(node.Name() + input + "_imm_b"); const auto tensor_imm_product_name = model_builder.GetUniqueName(node.Name() + input + "_imm_mul"); - ModelBuilder::Shape tensor_a_dimen; + Shape tensor_a_dimen; bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = input_is_nhwc; @@ -741,7 +756,7 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil output_is_nhwc); // Add - int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); AddBinaryOperator(ANEURALNETWORKS_ADD, model_builder, tensor_imm_product_name, tensor_b_name, @@ -812,7 +827,14 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return false; } } else if (op == "GlobalAveragePool" || op == "GlobalMaxPool") { - const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + Shape input_shape; + if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape info"; + return false; + } + + const auto input_size = input_shape.size(); if (input_size != 4) { LOGS_DEFAULT(VERBOSE) << "GlobalAveragePool/GlobalMaxPool Only rank-4 tensor is supported in " @@ -874,7 +896,7 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, static_cast(shaper[input][2])}; } - int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); std::vector input_indices; input_indices.push_back(operand_indices.at(input)); input_indices.push_back(model_builder.AddOperandFromScalar(onnx_pads[1])); @@ -1011,7 +1033,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, bias_idx_val = operand_indices.at(bias); } else { const auto weight_dimen = shaper[weight]; - ModelBuilder::Shape bias_dimen; + Shape bias_dimen; if (conv2d) bias_dimen = {weight_dimen[0]}; else @@ -1042,7 +1064,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, int32_t depthwiseMultiplier = shaper[weight][3] / group; input_indices.push_back(model_builder.AddOperandFromScalar(depthwiseMultiplier)); } - int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); // TODO support API 28 input_indices.push_back(model_builder.AddOperandFromScalar(use_nchw)); @@ -1151,7 +1173,14 @@ class SoftMaxOpBuilder : public BaseOpBuilder { bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const onnxruntime::Node& node) { - const auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + Shape input_shape; + if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape info"; + return false; + } + + const auto input_size = input_shape.size(); if (input_size != 2 && input_size != 4) { LOGS_DEFAULT(VERBOSE) << "SoftMax only support 2d/4d shape, input is " << input_size << "d shape"; @@ -1243,14 +1272,32 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const auto& op = node.OpType(); const auto& initializers(model_builder.GetInitializerTensors()); - if (GetShape(node.InputDefs()[0]->Shape()).size() != 2) { - LOGS_DEFAULT(VERBOSE) << "A must be 2D"; - return false; + Shape a_shape; + { + if (!GetShape(node.InputDefs()[0]->Shape(), a_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape A info"; + return false; + } + + if (a_shape.size() != 2) { + LOGS_DEFAULT(VERBOSE) << "A must be 2D"; + return false; + } } - if (GetShape(node.InputDefs()[1]->Shape()).size() != 2) { - LOGS_DEFAULT(VERBOSE) << "B must be 2D"; - return false; + Shape b_shape; + { + if (!GetShape(node.InputDefs()[1]->Shape(), b_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape B info"; + return false; + } + + if (b_shape.size() != 2) { + LOGS_DEFAULT(VERBOSE) << "B must be 2D"; + return false; + } } if (op == "MatMul") { // Only support A*B B is an initializer @@ -1280,8 +1327,13 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, } if (node.InputDefs().size() == 3) { - const auto b_shape = GetShape(node.InputDefs()[1]->Shape()); - const auto c_shape = GetShape(node.InputDefs()[2]->Shape()); + Shape c_shape; + if (!GetShape(node.InputDefs()[2]->Shape(), c_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape C info"; + return false; + } + if (c_shape.size() != 1 || c_shape[0] != (transB == 0 ? b_shape[1] : b_shape[0])) { LOGS_DEFAULT(VERBOSE) << "C of Gemm must be a vector of b_shape[0]" @@ -1333,7 +1385,7 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, if (node.InputDefs().size() == 2) { std::string bias = node.Name() + op + "_bias"; const auto& B_type = operand_types.at(input2).type; - ModelBuilder::Shape bias_dimen = {shaper[input2][0]}; + Shape bias_dimen = {shaper[input2][0]}; if (B_type == Type::TENSOR_FLOAT32) { float buffer[bias_dimen[0]]; for (uint32_t i = 0; i < bias_dimen[0]; i++) { @@ -1353,7 +1405,7 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, input_indices.push_back(operand_indices.at(input1)); // A input_indices.push_back(input_2_idx); // B input_indices.push_back(bias_idx); // C - int32_t fuse_code = model_builder.FindActivation(node, node.OutputDefs()[0]); + int32_t fuse_code = model_builder.FindActivation(node, *node.OutputDefs()[0]); input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); shaper.FC(input1, input2, output); @@ -1446,7 +1498,14 @@ class ConcatOpBuilder : public BaseOpBuilder { bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const onnxruntime::Node& node) { - auto input_size = GetShape(node.InputDefs()[0]->Shape()).size(); + Shape input_shape; + if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { + LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" + << node.OpType() << "] has no shape info"; + return false; + } + + const auto input_size = input_shape.size(); if (input_size > 4 || input_size == 0) { LOGS_DEFAULT(VERBOSE) << "Concat only supports up to 1-4d shape, input is " << input_size << "d shape"; From ab4a5d3978a2e8bd88f8207fbbaef2934bcfe661 Mon Sep 17 00:00:00 2001 From: gwang Date: Wed, 1 Jul 2020 22:04:07 -0700 Subject: [PATCH 09/11] Change the GetType and GetShape to log waring info inside to simplify the caller, remove some redundant onnxruntime namespace --- .../nnapi_builtin/builders/model_builder.cc | 12 +- .../nnapi_builtin/builders/model_builder.h | 12 +- .../nnapi_builtin/builders/op_builder.cc | 266 ++++++------------ .../nnapi/nnapi_builtin/builders/op_builder.h | 6 +- .../nnapi_builtin/nnapi_execution_provider.cc | 5 +- 5 files changed, 105 insertions(+), 196 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index f453bbdcb5407..33d6412da044b 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -15,7 +15,7 @@ namespace nnapi { using namespace android::nn::wrapper; using std::vector; -ModelBuilder::ModelBuilder(const onnxruntime::GraphViewer& graph_view) +ModelBuilder::ModelBuilder(const GraphViewer& graph_view) : nnapi_(NnApiImplementation()), graph_view_(graph_view) { GetAllInitializers(); op_builders_ = CreateOpBuilders(); @@ -25,8 +25,7 @@ int32_t ModelBuilder::GetAndroidSdkVer() const { return nnapi_ ? nnapi_->android_sdk_version : 0; } -bool ModelBuilder::IsNodeSupported( - const onnxruntime::Node& node) { +bool ModelBuilder::IsNodeSupported(const Node& node) { if (auto* op_builder = GetOpBuilder(node)) { return op_builder->IsOpSupported(*this, node); } else { @@ -34,8 +33,7 @@ bool ModelBuilder::IsNodeSupported( } } -bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, - const onnxruntime::GraphViewer& graph_view) { +bool IsValidSupportedNodesVec(const std::vector& supported_node_vec, const GraphViewer& graph_view) { if (!supported_node_vec.empty()) { if (supported_node_vec.size() == 1) { const auto& node_indices = graph_view.GetNodesInTopologicalOrder(); @@ -475,7 +473,7 @@ std::unique_ptr ModelBuilder::Compile() { return std::move(nnapi_model_); } -int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, const NodeArg& output) { +int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) { int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) { const auto& dst_node = it->GetNode(); @@ -508,7 +506,7 @@ int32_t ModelBuilder::FindActivation(const onnxruntime::Node& node, const NodeAr return fuse_code; } -IOpBuilder* ModelBuilder::GetOpBuilder(const onnxruntime::Node& node) { +IOpBuilder* ModelBuilder::GetOpBuilder(const Node& node) { if (!Contains(op_builders_, node.OpType())) return nullptr; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index eb21bf82e05b8..2b22736106645 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -28,7 +28,7 @@ class ModelBuilder { CPU_ONLY, // use CPU only }; - ModelBuilder(const onnxruntime::GraphViewer& graph_view); + ModelBuilder(const GraphViewer& graph_view); ~ModelBuilder() = default; std::vector> GetSupportedNodes(); @@ -44,7 +44,7 @@ class ModelBuilder { const std::vector& is_nhwc_vec); // Find if an output has a fuseable activation (Relu) - int32_t FindActivation(const onnxruntime::Node& node, const NodeArg& output); + int32_t FindActivation(const Node& node, const NodeArg& output); // Add an NNAPI scalar operand uint32_t AddOperandFromScalar(bool value); @@ -94,7 +94,7 @@ class ModelBuilder { const std::unordered_map& GetInitializerTensors() const { return initializers_; } - const onnxruntime::Graph& GetOnnxGraph() const { return graph_view_.GetGraph(); } + const Graph& GetOnnxGraph() const { return graph_view_.GetGraph(); } void RegisterNHWCOperand(const std::string& name); bool IsOperandNHWC(const std::string& name); @@ -110,7 +110,7 @@ class ModelBuilder { private: const NnApi* nnapi_{nullptr}; - const onnxruntime::GraphViewer& graph_view_; + const GraphViewer& graph_view_; std::unique_ptr nnapi_model_; uint32_t name_token_{0}; @@ -150,7 +150,7 @@ class ModelBuilder { uint32_t next_index_ = 0; - bool IsNodeSupported(const onnxruntime::Node& node); + bool IsNodeSupported(const Node& node); // Convert the onnx model to ANeuralNetworksModel void Prepare(); @@ -172,7 +172,7 @@ class ModelBuilder { const android::nn::wrapper::OperandType& operand_type, bool is_nhwc); - IOpBuilder* GetOpBuilder(const onnxruntime::Node& node); + IOpBuilder* GetOpBuilder(const Node& node); }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 9a89f6946e27a..c6fc0dc336b97 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -124,9 +124,11 @@ void AddBinaryOperator(int32_t op_type, model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -bool GetType(const ONNX_NAMESPACE::TypeProto* type_proto, int32_t& type) { +bool GetType(const NodeArg& node_arg, int32_t& type) { type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; + const auto* type_proto = node_arg.TypeAsProto(); if (!type_proto || !type_proto->has_tensor_type() || !type_proto->tensor_type().has_elem_type()) { + LOGS_DEFAULT(WARNING) << "NodeArg [" << node_arg.Name() << "] has no input type"; return false; } @@ -134,11 +136,14 @@ bool GetType(const ONNX_NAMESPACE::TypeProto* type_proto, int32_t& type) { return true; } -bool GetShape(const ONNX_NAMESPACE::TensorShapeProto* shape_proto, Shape& shape) { +bool GetShape(const NodeArg& node_arg, Shape& shape) { shape.clear(); + const auto* shape_proto = node_arg.Shape(); - if (!shape_proto) + if (!shape_proto) { + LOGS_DEFAULT(WARNING) << "NodeArg [" << node_arg.Name() << "] has no shape info"; return false; + } // NNAPI uses 0 for dynamic dimension, which is the default value for dim.dim_value() for (const auto& dim : shape_proto->dim()) @@ -260,28 +265,26 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, class BaseOpBuilder : public IOpBuilder { public: virtual ~BaseOpBuilder() = default; - virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) override {} + virtual void AddInitializersToSkip(ModelBuilder& /* model_builder */, const Node& /* node */) override {} - bool IsOpSupported(ModelBuilder& model_builder, const onnxruntime::Node& node) override final; + bool IsOpSupported(ModelBuilder& model_builder, const Node& node) override final; - void AddToModelBuilder(ModelBuilder& model_builder, const onnxruntime::Node& node) override final; + void AddToModelBuilder(ModelBuilder& model_builder, const Node& node) override final; protected: - virtual bool IsOpSupportedImpl(ModelBuilder& model_builder, const onnxruntime::Node& node); + virtual bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node); virtual int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) const { return 27; } + const Node& /* node */) const { return 27; } - virtual bool HasSupportedInputs(const onnxruntime::Node& node); + virtual bool HasSupportedInputs(const Node& node); - virtual void AddToModelBuilderImpl(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; + virtual void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) = 0; - bool HasExternalInitializer(ModelBuilder& model_builder, const onnxruntime::Node& node); + bool HasExternalInitializer(ModelBuilder& model_builder, const Node& node); }; -bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, const Node& node) { #ifdef __ANDROID__ int32_t android_sdk_ver = model_builder.GetAndroidSdkVer(); int32_t required_sdk_ver = GetMinSupportedSdkVer(model_builder, node); @@ -303,14 +306,12 @@ bool BaseOpBuilder::IsOpSupported(ModelBuilder& model_builder, return IsOpSupportedImpl(model_builder, node); } -bool BaseOpBuilder::HasSupportedInputs(const onnxruntime::Node& node) { +bool BaseOpBuilder::HasSupportedInputs(const Node& node) { // We only check the type of input 0 by default // specific op builder can override this int32_t input_type; - if (!GetType(node.InputDefs()[0]->TypeAsProto(), input_type)) { - LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] input 0 has no input type"; + if (!GetType(*node.InputDefs()[0], input_type)) return false; - } if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() @@ -322,13 +323,11 @@ bool BaseOpBuilder::HasSupportedInputs(const onnxruntime::Node& node) { return true; } -bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) { +bool BaseOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& /* node */) { return true; } -void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, const Node& node) { ORT_ENFORCE(IsOpSupported(model_builder, node), "Unsupported operator " + node.OpType()); @@ -337,8 +336,7 @@ void BaseOpBuilder::AddToModelBuilder(ModelBuilder& model_builder, << "] type: [" << node.OpType() << "] was added"; } -bool BaseOpBuilder::HasExternalInitializer(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +bool BaseOpBuilder::HasExternalInitializer(ModelBuilder& model_builder, const Node& node) { const auto& initializers(model_builder.GetOnnxGraph().GetAllInitializedTensors()); for (const auto* node_arg : node.InputDefs()) { const auto& input_name(node_arg->Name()); @@ -363,16 +361,13 @@ bool BaseOpBuilder::HasExternalInitializer(ModelBuilder& model_builder, class BinaryOpBuilder : public BaseOpBuilder { private: - int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, - const onnxruntime::Node& node) const override; + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const Node& node) const override; private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) const { +int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& node) const { const auto& op(node.OpType()); if (op == "Sub" || op == "Div") { return 28; @@ -381,8 +376,7 @@ int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */ return 27; } -void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { const auto& op(node.OpType()); int32_t op_code; if (op == "Add") @@ -432,12 +426,10 @@ void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ReluOpBuilder : public BaseOpBuilder { private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -465,26 +457,19 @@ void ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class TransposeOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) const override { + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { return 28; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) { +bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { Shape input_shape; - if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape info"; + if (!GetShape(*node.InputDefs()[0], input_shape)) return false; - } const auto input_size = input_shape.size(); if (input_size > 4 || input_size == 0) { @@ -496,8 +481,7 @@ bool TransposeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); auto input = node.InputDefs()[0]->Name(); @@ -536,24 +520,19 @@ void TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ReshapeOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void ReshapeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } -bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& perm_name = node.InputDefs()[1]->Name(); if (!Contains(initializers, perm_name)) { @@ -562,11 +541,8 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, } Shape input_shape; - if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape info"; + if (!GetShape(*node.InputDefs()[0], input_shape)) return false; - } if (input_shape.size() > 4 || input_shape.empty()) { LOGS_DEFAULT(VERBOSE) << "Reshape only supports up to 1-4d shape, input is " @@ -589,8 +565,7 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -641,19 +616,15 @@ void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class BatchNormalizationOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { // skip everything except input0 for BatchNormalization model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); // scale model_builder.AddInitializerToSkip(node.InputDefs()[2]->Name()); // B @@ -661,8 +632,7 @@ void BatchNormalizationOpBuilder::AddInitializersToSkip(ModelBuilder& model_buil model_builder.AddInitializerToSkip(node.InputDefs()[4]->Name()); //var } -bool BatchNormalizationOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +bool BatchNormalizationOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { if (node.OutputDefs().size() != 1) { LOGS_DEFAULT(VERBOSE) << "Your onnx model may be in training mode, please export " "it in test mode."; @@ -694,8 +664,7 @@ bool BatchNormalizationOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); @@ -771,20 +740,16 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil class PoolOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) const override { + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { return 28; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) { +bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { const auto& op = node.OpType(); if (op == "AveragePool" || op == "MaxPool") { NodeAttrHelper helper(node); @@ -828,11 +793,8 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, } } else if (op == "GlobalAveragePool" || op == "GlobalMaxPool") { Shape input_shape; - if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape info"; + if (!GetShape(*node.InputDefs()[0], input_shape)) return false; - } const auto input_size = input_shape.size(); if (input_size != 4) { @@ -846,8 +808,7 @@ bool PoolOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -924,25 +885,20 @@ void PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ConvOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { // skip the weight for conv as we need to transpose model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); } -bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { NodeAttrHelper helper(node); if (helper.Get("auto_pad", "NOTSET") != "NOTSET") { LOGS_DEFAULT(VERBOSE) << "SAME_LOWER auto_pad is not supported"; @@ -969,8 +925,7 @@ bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -1096,20 +1051,16 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class CastOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) const override { + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { return 29; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) { +bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { NodeAttrHelper helper(node); auto to = helper.Get("to", 0); if (to != ONNX_NAMESPACE::TensorProto::FLOAT && @@ -1121,8 +1072,7 @@ bool CastOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); NodeAttrHelper helper(node); @@ -1159,26 +1109,19 @@ void CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class SoftMaxOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const onnxruntime::Node& /* node */) const override { + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { return 29; } - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) { +bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { Shape input_shape; - if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape info"; + if (!GetShape(*node.InputDefs()[0], input_shape)) return false; - } const auto input_size = input_shape.size(); if (input_size != 2 && input_size != 4) { @@ -1189,8 +1132,7 @@ bool SoftMaxOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -1226,12 +1168,10 @@ void SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class IdentityOpBuilder : public BaseOpBuilder { private: - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { // Identity is not really going to do anything // Just register the dimension and type, with same index and new name auto& shaper(model_builder.GetShaper()); @@ -1256,29 +1196,22 @@ void IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class GemmOpBuilder : public BaseOpBuilder { public: - void AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { const auto& op = node.OpType(); const auto& initializers(model_builder.GetInitializerTensors()); Shape a_shape; { - if (!GetShape(node.InputDefs()[0]->Shape(), a_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape A info"; + if (!GetShape(*node.InputDefs()[0], a_shape)) return false; - } if (a_shape.size() != 2) { LOGS_DEFAULT(VERBOSE) << "A must be 2D"; @@ -1288,11 +1221,8 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, Shape b_shape; { - if (!GetShape(node.InputDefs()[1]->Shape(), b_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape B info"; + if (!GetShape(*node.InputDefs()[1], b_shape)) return false; - } if (b_shape.size() != 2) { LOGS_DEFAULT(VERBOSE) << "B must be 2D"; @@ -1328,11 +1258,8 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, if (node.InputDefs().size() == 3) { Shape c_shape; - if (!GetShape(node.InputDefs()[2]->Shape(), c_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape C info"; + if (!GetShape(*node.InputDefs()[2], c_shape)) return false; - } if (c_shape.size() != 1 || c_shape[0] != (transB == 0 ? b_shape[1] : b_shape[0])) { @@ -1348,8 +1275,7 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, return true; } -void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { const auto& op = node.OpType(); if (op == "MatMul") { model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); @@ -1361,8 +1287,7 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, } } -void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { const auto& op = node.OpType(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); @@ -1420,15 +1345,12 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class UnaryOpBuilder : public BaseOpBuilder { private: - int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, - const onnxruntime::Node& node) const override; + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const Node& node) const override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) const { +int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& node) const { const auto& op(node.OpType()); if (op == "Abs" || op == "Exp" || @@ -1442,8 +1364,7 @@ int32_t UnaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, return 27; } -void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -1489,21 +1410,15 @@ void UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, class ConcatOpBuilder : public BaseOpBuilder { private: - bool IsOpSupportedImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - void AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; -bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, - const onnxruntime::Node& node) { +bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { Shape input_shape; - if (!GetShape(node.InputDefs()[0]->Shape(), input_shape)) { - LOGS_DEFAULT(WARNING) << "Node [" << node.Name() << "] type [" - << node.OpType() << "] has no shape info"; + if (!GetShape(*node.InputDefs()[0], input_shape)) return false; - } const auto input_size = input_shape.size(); if (input_size > 4 || input_size == 0) { @@ -1515,8 +1430,7 @@ bool ConcatOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, return true; } -void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, - const onnxruntime::Node& node) { +void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index 8070e1c968b1a..d800a59b12c80 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -13,14 +13,14 @@ class IOpBuilder { virtual ~IOpBuilder() = default; // Check if an operator is supported - virtual bool IsOpSupported(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; + virtual bool IsOpSupported(ModelBuilder& model_builder, const Node& node) = 0; // Check if the initializers of this operator need preprocess // which will not be copied - virtual void AddInitializersToSkip(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; + virtual void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) = 0; // Add the operator to NNAPI model - virtual void AddToModelBuilder(ModelBuilder& model_builder, const onnxruntime::Node& node) = 0; + virtual void AddToModelBuilder(ModelBuilder& model_builder, const Node& node) = 0; }; // Generate a lookup table with IOpBuilder delegates diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index f284dd1851535..b0c5a74d47aca 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -181,11 +181,8 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorBody(); - onnxruntime::Model model(graph_body.Name(), true, ModelMetaData(), PathString(), - IOnnxRuntimeOpSchemaRegistryList(), graph_body.DomainToVersionMap(), - std::vector(), *GetLogger()); + const Graph& graph_body = func_body->Body(); { onnxruntime::GraphViewer graph_viewer(graph_body); nnapi::ModelBuilder builder(graph_viewer); From 5366f06682ddfd928fb8645eae8e376a66dcfdc9 Mon Sep 17 00:00:00 2001 From: gwang Date: Thu, 2 Jul 2020 15:41:34 -0700 Subject: [PATCH 10/11] add squeeze op support, some more code style clean up --- .../nnapi_builtin/builders/op_builder.cc | 98 ++++++++++++++++--- .../nnapi/nnapi_builtin/builders/shaper.cc | 36 +++++++ .../nnapi/nnapi_builtin/builders/shaper.h | 37 +++---- 3 files changed, 137 insertions(+), 34 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index c6fc0dc336b97..10864be333012 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -188,7 +188,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, const float* src = GetTensorFloatData(tensor); float* buffer = new float[Product(shape)]; - const OperandType operandType(type, dest_shape); + const OperandType operand_type(type, dest_shape); for (uint32_t out = 0; out < out_t; out++) { for (uint32_t in = 0; in < in_t; in++) { for (uint32_t h = 0; h < h_t; h++) { @@ -217,7 +217,7 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, } } - auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operandType); + auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); delete[] buffer; return operand_idx; } @@ -244,7 +244,7 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, auto x_t = shape[0], y_t = shape[1]; Shape dest_shape = {y_t, x_t}; - const OperandType operandType(type, dest_shape); + const OperandType operand_type(type, dest_shape); const float* src = GetTensorFloatData(tensor); float* buffer = new float[Product(shape)]; for (uint32_t x = 0; x < x_t; x++) { @@ -252,7 +252,7 @@ uint32_t AddInitializerTransposed(ModelBuilder& model_builder, buffer[y * x_t + x] = src[x * y_t + y]; } } - auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operandType); + auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); delete[] buffer; return operand_idx; @@ -572,7 +572,6 @@ void ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& initializers(model_builder.GetInitializerTensors()); auto input = node.InputDefs()[0]->Name(); - if (model_builder.IsOperandNHWC(input)) { // We want to transpose nhwc operand back to nchw before reshape const auto& nhwc_input = node.InputDefs()[0]->Name(); @@ -711,10 +710,10 @@ void BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil shaper.AddShape(tensor_a_name, tensor_a_dimen); shaper.AddShape(tensor_b_name, tensor_a_dimen); - const OperandType operandType_a(operand_types.at(input).type, tensor_a_dimen); - model_builder.AddOperandFromPersistMemoryBuffer(tensor_a_name, a.data(), operandType_a); - const OperandType operandType_b(operand_types.at(input).type, tensor_a_dimen); - model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), operandType_b); + const OperandType a_operand_type(operand_types.at(input).type, tensor_a_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(tensor_a_name, a.data(), a_operand_type); + const OperandType b_operand_type(operand_types.at(input).type, tensor_a_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), b_operand_type); // Mul AddBinaryOperator(ANEURALNETWORKS_MUL, @@ -1000,9 +999,9 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod for (uint32_t i = 0; i < buffer.size(); i++) { buffer[i] = 0.f; } - OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); + OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen); bias_idx_val = model_builder.AddOperandFromPersistMemoryBuffer( - bias, buffer.data(), operandType); + bias, buffer.data(), bias_operand_type); } else { ORT_THROW("Unknown weight type " + TypeToStr(weight_type)); } @@ -1316,9 +1315,9 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod for (uint32_t i = 0; i < bias_dimen[0]; i++) { buffer[i] = 0.f; } - OperandType operandType(Type::TENSOR_FLOAT32, bias_dimen); + OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen); bias_idx = model_builder.AddOperandFromPersistMemoryBuffer( - bias, &buffer[0], operandType); + bias, &buffer[0], bias_operand_type); } else { ORT_THROW("Unknown weight type " + TypeToStr(B_type)); } @@ -1502,6 +1501,78 @@ void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N #pragma endregion +#pragma region op_squeeze + +class SqueezeOpBuilder : public BaseOpBuilder { + private: + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { + return 28; + } + + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; +}; + +bool SqueezeOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + Shape input_shape; + if (!GetShape(*node.InputDefs()[0], input_shape)) + return false; + + const auto input_size = input_shape.size(); + if (input_size > 4 || input_size == 0) { + LOGS_DEFAULT(VERBOSE) << "Squeeze only supports 1-4d shape, input is " + << input_size << "d shape"; + return false; + } + + return true; +} + +void SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + + auto input = node.InputDefs()[0]->Name(); + if (model_builder.IsOperandNHWC(input)) { + // We want to transpose nhwc operand back to nchw before squeeze + const auto& nhwc_input = node.InputDefs()[0]->Name(); + if (!model_builder.GetNCHWOperand(nhwc_input, input)) { + input = model_builder.GetUniqueName(nhwc_input + "_nhwc_to_nchw"); + TransposeNHWCToNCHW(model_builder, nhwc_input, input); + } + } + + NodeAttrHelper helper(node); + vector axes = helper.Get("axes", vector()); + auto input_dims = shaper[input].size(); + for (auto& axis : axes) { + if (axis < 0) + axis += input_dims; + } + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); // input + + if (!axes.empty()) { + const auto axes_name = model_builder.GetUniqueName(node.Name() + input + "_axes"); + Shape axes_dimen = {static_cast(axes.size())}; + shaper.AddShape(axes_name, axes_dimen); + const OperandType axes_operand_type(Type::TENSOR_INT32, axes_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(axes_name, axes.data(), axes_operand_type); + input_indices.push_back(operand_indices.at(axes_name)); // axes + } + + const auto& output = node.OutputDefs()[0]->Name(); + shaper.Squeeze(input, axes, output); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + model_builder.AddOperation(ANEURALNETWORKS_SQUEEZE, input_indices, {output}, + {output_operand_type}, {false}); +} + +#pragma endregion + #pragma region CreateOpBuilders std::unordered_map> @@ -1554,6 +1625,7 @@ CreateOpBuilders() { } op_map.emplace("Concat", std::make_shared()); + op_map.emplace("Squeeze", std::make_shared()); return op_map; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc index 3f9a2251d7c6e..680930795c89c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc @@ -1,4 +1,5 @@ #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" + #include "helper.h" #include "shaper.h" @@ -378,6 +379,41 @@ void Shaper::Concat(const std::vector& input_names, } } +void Shaper::Squeeze(const std::string& input_name, + const std::vector& axes, + const std::string& output_name) { + std::vector input_dimen = shape_map_.at(input_name); + int32_t input_size = input_dimen.size(); + size_t axes_size = axes.size(); + std::unordered_set axes_to_be_squeezed; + if (axes_size == 0) { + for (int32_t idx = 0; idx < input_size; ++idx) { + if (input_dimen[idx] == 1) + axes_to_be_squeezed.insert(idx); + } + } else { + for (const auto& axis : axes) + axes_to_be_squeezed.insert(axis); + } + + // Make output dimensions + std::vector output_dimen; + output_dimen.reserve(input_size - axes_to_be_squeezed.size()); + for (int32_t i = 0; i < input_size; i++) { + if (!Contains(axes_to_be_squeezed, i)) + output_dimen.push_back(input_dimen[i]); + } + + shape_map_[output_name] = output_dimen; + + if (!shaper_finalized_) { + shape_ops_.push_back( + [input_name, axes, output_name](Shaper& shaper) { + shaper.Squeeze(input_name, axes, output_name); + }); + } +} + void Shaper::AddShape(const std::string& name, const Shape& shape) { shape_map_[name] = shape; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h index 862c6134d9034..634f02d1a43e4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h @@ -9,6 +9,9 @@ class Shaper { using Shape = std::vector; void AddShape(const std::string& name, const Shape& shape); + inline const Shape& operator[](const std::string& key) const { + return shape_map_.at(key); + } void Conv(const std::string& input_name, const std::string& weight_name, @@ -33,23 +36,19 @@ class Shaper { bool nchw, const std::string& output_name); - void Reshape(const std::string& input_name, - const std::vector& shape, - const std::string& output_name); - void Transpose(const std::string& input_name, - const std::vector& perm, - const std::string& output_name); - void Eltwise(const std::string& input1_name, const std::string& input2_name, - const std::string& output_name); - void Identity(const std::string& input_name, - const std::string& output_name); - void FC(const std::string& input1_name, - const std::string& input2_name, - const std::string& output_name); - - void Concat(const std::vector& input_names, - const int32_t axis, - const std::string& output_name); + void Reshape(const std::string& input_name, const std::vector& shape, const std::string& output_name); + + void Transpose(const std::string& input_name, const std::vector& perm, const std::string& output_name); + + void Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + + void Identity(const std::string& input_name, const std::string& output_name); + + void FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); + + void Concat(const std::vector& input_names, const int32_t axis, const std::string& output_name); + + void Squeeze(const std::string& input, const std::vector& axes, const std::string& output); // If the shape of certain input is dynamic // Use the following 2 functions to update the particular shape @@ -61,10 +60,6 @@ class Shaper { // is converted to NNAPI void Finalize() { shaper_finalized_ = true; } - inline const Shape& operator[](const std::string& key) const { - return shape_map_.at(key); - } - void Clear(); private: From 5f6d88a86c94e9d917838a9328e33aec01e3476a Mon Sep 17 00:00:00 2001 From: gwang Date: Mon, 6 Jul 2020 14:59:32 -0700 Subject: [PATCH 11/11] fix a bug where duplicate output can be added to a subgraph, some other minor logging changes --- .../nnapi/nnapi_builtin/builders/model_builder.cc | 9 +++------ .../nnapi/nnapi_builtin/nnapi_execution_provider.cc | 4 +--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 33d6412da044b..c92aa2a11d7a5 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -268,7 +268,7 @@ void ModelBuilder::RegisterModelInputs() { } const auto* shape_proto = node_arg->Shape(); - ORT_ENFORCE(shape_proto != nullptr, "shape_proto cannot be null"); + ORT_ENFORCE(shape_proto != nullptr, "shape_proto cannot be null for input: " + input_name); Shaper::Shape shape; for (const auto& dim : shape_proto->dim()) { @@ -279,8 +279,7 @@ void ModelBuilder::RegisterModelInputs() { Type type = Type::TENSOR_FLOAT32; const auto* type_proto = node_arg->TypeAsProto(); if (!type_proto || !type_proto->tensor_type().has_elem_type()) { - ORT_THROW("The input of graph doesn't have elem_type: " + - input_name); + ORT_THROW("The input of graph doesn't have elem_type: " + input_name); } else { switch (type_proto->tensor_type().elem_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: @@ -331,9 +330,7 @@ void ModelBuilder::RegisterModelShaper() { uint32_t ModelBuilder::AddNewOperand(const std::string& name, const OperandType& operand_type, bool is_nhwc) { - THROW_ON_ERROR(nnapi_->ANeuralNetworksModel_addOperand( - nnapi_model_->model_, &operand_type.operandType)); - auto idx = next_index_++; + auto idx = AddNewNNAPIOperand(operand_type); RegisterOperand(name, idx, operand_type, is_nhwc); return idx; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index b0c5a74d47aca..a2fe6b345e3c2 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -142,9 +142,7 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view for (auto it = fused_outputs.begin(), end = fused_outputs.end(); it != end; ++it) { if (all_node_inputs.find(it->first->Name()) != all_node_inputs.end()) { outputs.insert(std::pair(it->second, it->first)); - } - - if (std::find(graph_outputs.begin(), graph_outputs.end(), it->first) != graph_outputs.end()) { + } else if (std::find(graph_outputs.begin(), graph_outputs.end(), it->first) != graph_outputs.end()) { outputs.insert(std::pair(it->second, it->first)); } }