diff --git a/cgmanifest.json b/cgmanifest.json index 782db27f7c8d0..e2b2c73a60a57 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -450,7 +450,7 @@ { "component": { "git": { - "commitHash": "3f0f9802553944b75015aad098d856b2d17220df", + "commitHash": "a11f5002af58a03d5902b13ef65c84cedb499024", "repositoryUrl": "https://github.com/microsoft/FeaturizersLibrary.git" }, "type": "git" diff --git a/cmake/external/featurizers.cmake b/cmake/external/featurizers.cmake index ac7f2432db921..acc9b25c564d1 100644 --- a/cmake/external/featurizers.cmake +++ b/cmake/external/featurizers.cmake @@ -3,15 +3,14 @@ # This source code should not depend on the onnxruntime and may be built independently set(featurizers_URL "https://github.com/microsoft/FeaturizersLibrary.git") -set(featurizers_TAG "3f0f9802553944b75015aad098d856b2d17220df") +set(featurizers_TAG "a11f5002af58a03d5902b13ef65c84cedb499024") set(featurizers_pref FeaturizersLibrary) set(featurizers_ROOT ${PROJECT_SOURCE_DIR}/external/${featurizers_pref}) set(featurizers_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/${featurizers_pref}) -# Only due to GIT_CONFIG -# Uncoment UPDATE_COMMAND if you work locally -# on the featurizers so cmake does not undo your changes. +# Windows required git config setting +# and an a switch whether we link to MSVCRT statically or dynamically if (WIN32) ExternalProject_Add(featurizers_lib PREFIX ${featurizers_pref} @@ -24,7 +23,6 @@ if (WIN32) SOURCE_SUBDIR src/Featurizers BINARY_DIR ${featurizers_BINARY_DIR} CMAKE_ARGS -Dfeaturizers_MSVC_STATIC_RUNTIME=${onnxruntime_MSVC_STATIC_RUNTIME} -# UPDATE_COMMAND "" INSTALL_COMMAND "" ) else() @@ -37,7 +35,6 @@ else() SOURCE_SUBDIR src/Featurizers BINARY_DIR ${featurizers_BINARY_DIR} CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON -# UPDATE_COMMAND "" INSTALL_COMMAND "" ) endif() diff --git a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc index d6db03b3f6c30..5c12ef18bce8f 100644 --- a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc +++ b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc @@ -34,7 +34,12 @@ using ONNX_NAMESPACE::OPTIONAL; // Forward declarations static void RegisterCatImputerFeaturizerVer1(); static void RegisterDateTimeFeaturizerVer1(); +static void RegisterImputationMarkerFeaturizerVer1(); +static void RegisterLabelEncoderFeaturizerVer1(); static void RegisterMaxAbsScalarFeaturizerVer1(); +static void RegisterMinMaxScalarFeaturizerVer1(); +static void RegisterMissingDummiesFeaturizerVer1(); +static void RegisterRobustScalarFeaturizerVer1(); static void RegisterStringFeaturizerVer1(); // ---------------------------------------------------------------------- @@ -43,7 +48,12 @@ static void RegisterStringFeaturizerVer1(); void RegisterMSFeaturizersSchemas() { RegisterCatImputerFeaturizerVer1(); RegisterDateTimeFeaturizerVer1(); + RegisterImputationMarkerFeaturizerVer1(); + RegisterLabelEncoderFeaturizerVer1(); RegisterMaxAbsScalarFeaturizerVer1(); + RegisterMinMaxScalarFeaturizerVer1(); + RegisterMissingDummiesFeaturizerVer1(); + RegisterRobustScalarFeaturizerVer1(); RegisterStringFeaturizerVer1(); } @@ -76,7 +86,7 @@ void RegisterCatImputerFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -87,6 +97,10 @@ void RegisterCatImputerFeaturizerVer1() { "Output", "No information is available", "T") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "T", {"tensor(float)", "tensor(double)", "tensor(string)"}, @@ -94,10 +108,9 @@ void RegisterCatImputerFeaturizerVer1() { .TypeAndShapeInferenceFunction( [](ONNX_NAMESPACE::InferenceContext& ctx) { propagateElemTypeFromInputToOutput(ctx, 1, 0); - if (!hasNInputShapes(ctx, 1)) { - return; + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); } - propagateShapeFromInputToOutput(ctx, 1, 0); }); } @@ -144,7 +157,7 @@ void RegisterDateTimeFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -171,6 +184,10 @@ void RegisterDateTimeFeaturizerVer1() { .Output(18, "dayOfWeekLabel", "No information available", "OutputT3") .Output(19, "holidayName", "No information available", "OutputT3") .Output(20, "isPaidTimeOff", "No information available", "OutputT1") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "OutputT0", {"tensor(int32)"}, @@ -189,30 +206,156 @@ void RegisterDateTimeFeaturizerVer1() { "No information is available") .TypeAndShapeInferenceFunction( [](ONNX_NAMESPACE::InferenceContext& ctx) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(1)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(2)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(3)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(4)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(5)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(6)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(7)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(8)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(9)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(10)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(11)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT16); - ctx.getOutputType(12)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(13)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(14)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - ctx.getOutputType(15)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_INT32); - ctx.getOutputType(16)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(17)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(18)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(19)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_STRING); - ctx.getOutputType(20)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - - for (size_t i = 0; i < ctx.getNumOutputs(); ++i) { - *ctx.getOutputType(i)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + const bool has_shape = hasInputShape(ctx, 1); + for (int output = 0; output < 21; ++output) { + switch (output) { + case 0: + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, output); + break; + case 1: // fall through + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + case 9: + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, output); + break; + case 10: // fall through + case 11: + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT16, output); + break; + case 12: // fall through + case 13: + case 14: + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, output); + break; + case 15: + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT32, output); + break; + case 16: + case 17: + case 18: + case 19: + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, output); + break; + case 20: + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT8, output); + break; + default: + assert(false); + break; + } + if (has_shape) { + propagateShapeFromInputToOutput(ctx, 1, output); + } + } + }); +} + +void RegisterImputationMarkerFeaturizerVer1() { + static const char* doc = R"DOC( + Returns true if the input is null, false if it is not. + + C++-style pseudo signature: + bool execute(std::float_t const &value); + bool execute(std::double_t const &value); + template bool execute(std::optional const &value); + + Examples: + 3.0 -> false + NaN -> true + "foo" -> false + std::optional() -> true + std::optional("bar") -> false + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(ImputationMarkerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(bool)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(float)", "tensor(double)", "tensor(string)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_BOOL, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + }); +} + +void RegisterLabelEncoderFeaturizerVer1() { + static const char* doc = R"DOC( + Returns a unique id for the input based on all values encountered during training. + + C++-style pseudo signature: + template std::uint32_t execute(T const &value); + + Examples: + Assuming the training data of ["A", "B", "C"]... + + execute("A") -> 1 + execute("B") -> 2 + execute("C") -> 3 + execute("This value was not seen during training") -> 0 + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(LabelEncoderTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(uint32)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT32, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); } }); } @@ -242,7 +385,7 @@ void RegisterMaxAbsScalarFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -253,6 +396,10 @@ void RegisterMaxAbsScalarFeaturizerVer1() { "Output", "No information is available", "OutputT") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "InputT", {"tensor(int8)", "tensor(int16)", "tensor(uint8)", "tensor(uint16)", "tensor(float)", "tensor(int32)", "tensor(int64)", "tensor(uint32)", "tensor(uint64)", "tensor(double)"}, @@ -269,16 +416,193 @@ void RegisterMaxAbsScalarFeaturizerVer1() { input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT8 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, 0); } else if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT64 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT32 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT64 || input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) { - ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_DOUBLE); + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0); + } else { + fail_type_inference("input 1 is expected to have a accepted type"); + } + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); } + }); +} + +void RegisterMinMaxScalarFeaturizerVer1() { + static const char* doc = R"DOC( + Scales input based on the scale that results from the minimum and maximum values encountered + during training. + + C++-style pseudo signature: + template std::double_t(T const &value); + + Examples: + Given the training data [1, 2, 3, 4, 5]; + min: 1 + max: 5 + scale ( - ): 4 + + execute(2) = 2 / 4 + execute(20) = 20 / 4 + )DOC"; - *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + MS_FEATURIZERS_OPERATOR_SCHEMA(MinMaxScalarTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(double)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + }); +} + +void RegisterMissingDummiesFeaturizerVer1() { + static const char* doc = R"DOC( + Returns 1 if the input is null, 0 if it is not. + + C++-style pseudo signature: + std::int8_t execute(std::float_t const &value); + std::int8_t execute(std::double_t const &value); + template std::int8_t execute(T const &value); + + Examples: + 1.0 -> 0 + NaN -> 1 + "foo" -> 0 + std::optional() -> 1 + std::optional("bar") -> 0 + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(MissingDummiesTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "tensor(int8)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(float)", "tensor(double)", "tensor(string)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_INT8, 0); + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } + }); +} + +void RegisterRobustScalarFeaturizerVer1() { + static const char* doc = R"DOC( + MinMaxScalarEstimator + centering? + + C++-style pseudo signature: + TODO + + Examples: + TODO + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(RobustScalarTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "InputT") + .Output( + 0, + "Output", + "No information is available", + "OutputT") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "InputT", + {"tensor(int8)", "tensor(int16)", "tensor(uint8)", "tensor(uint16)", "tensor(float)", "tensor(int32)", "tensor(int64)", "tensor(uint32)", "tensor(uint64)", "tensor(double)"}, + "No information is available") + .TypeConstraint( + "OutputT", + {"tensor(float)", "tensor(double)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + auto input_elem_type = ctx.getInputType(1)->tensor_type().elem_type(); + if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT8 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT16 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, 0); + } else if (input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_INT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT32 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_UINT64 || + input_elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) { + ctx.getOutputType(0)->mutable_tensor_type()->set_elem_type(ONNX_NAMESPACE::TensorProto_DataType_DOUBLE); + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_DOUBLE, 0); + } else { + fail_type_inference("input 1 is expected to have a accepted type"); + } + if (hasInputShape(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 1, 0); + } }); } @@ -302,7 +626,7 @@ void RegisterStringFeaturizerVer1() { 0, "State", "State generated during training that is used for prediction", - "tensor(uint8)") + "T0") .Input( 1, "Input", @@ -313,6 +637,10 @@ void RegisterStringFeaturizerVer1() { "Output", "No information is available", "tensor(string)") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") .TypeConstraint( "InputT", {"tensor(int8)", "tensor(int16)", "tensor(int32)", "tensor(int64)", "tensor(uint8)", "tensor(uint16)", "tensor(uint32)", "tensor(uint64)", "tensor(float)", "tensor(double)", "tensor(bool)", "tensor(string)"}, @@ -320,8 +648,7 @@ void RegisterStringFeaturizerVer1() { .TypeAndShapeInferenceFunction( [](ONNX_NAMESPACE::InferenceContext& ctx) { propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_STRING, 0); - - *ctx.getOutputType(0)->mutable_tensor_type()->mutable_shape() = ctx.getInputType(1)->tensor_type().shape(); + propagateShapeFromInputToOutput(ctx, 1, 0); }); } diff --git a/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc b/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc index 7aa53d734e298..b2adb9108f8b9 100644 --- a/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/cat_imputer_transformer.cc @@ -3,6 +3,7 @@ #include "core/common/common.h" #include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" #include "core/framework/op_kernel.h" #include "Featurizers/CatImputerFeaturizer.h" @@ -26,13 +27,9 @@ inline nonstd::optional PreprocessOptional(std::string value) { return value.empty() ? nonstd::optional() : nonstd::optional(std::move(value)); } -template -class CatImputerTransformer final : public OpKernel { - public: - explicit CatImputerTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { +template +struct CatImputerTransformerImpl { + void operator()(OpKernelContext* ctx) const { // Create the transformer Microsoft::Featurizer::Featurizers::CatImputerTransformer transformer( [ctx](void) { @@ -57,40 +54,32 @@ class CatImputerTransformer final : public OpKernel { for (int64_t i = 0; i < length; ++i) { output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); } - - return Status::OK(); } }; -ONNX_OPERATOR_TYPED_KERNEL_EX( - CatImputerTransformer, - kMSFeaturizersDomain, - 1, - float, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - CatImputerTransformer); +class CatImputerTransformer final : public OpKernel { + public: + explicit CatImputerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } -ONNX_OPERATOR_TYPED_KERNEL_EX( - CatImputerTransformer, - kMSFeaturizersDomain, - 1, - double, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - CatImputerTransformer); + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; -ONNX_OPERATOR_TYPED_KERNEL_EX( +ONNX_OPERATOR_KERNEL_EX( CatImputerTransformer, kMSFeaturizersDomain, 1, - string, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - CatImputerTransformer); + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + CatImputerTransformer); } // namespace featurizers } // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc b/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc index 1ac583b0e5b76..0e25e58fac68e 100644 --- a/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/date_time_transformer.cc @@ -115,7 +115,7 @@ ONNX_OPERATOR_KERNEL_EX( 1, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) .TypeConstraint("T1", DataTypeImpl::GetTensorType()), DateTimeTransformer); diff --git a/onnxruntime/featurizers_ops/cpu/imputation_marker_transformer.cc b/onnxruntime/featurizers_ops/cpu/imputation_marker_transformer.cc new file mode 100644 index 0000000000000..a0f36a943ff01 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/imputation_marker_transformer.cc @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/ImputationMarkerFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +inline float const& PreprocessOptional(float const& value) { return value; } +inline double const& PreprocessOptional(double const& value) { return value; } +inline nonstd::optional PreprocessOptional(std::string value) { + return value.empty() ? nonstd::optional() : nonstd::optional(std::move(value)); +} + +template +struct ImputationMarkerTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::ImputationMarkerTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::ImputationMarkerTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + bool* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); + } + } +}; + +class ImputationMarkerTransformer final : public OpKernel { + public: + explicit ImputationMarkerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + ImputationMarkerTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + ImputationMarkerTransformer); +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/label_encoder_transformer.cc b/onnxruntime/featurizers_ops/cpu/label_encoder_transformer.cc new file mode 100644 index 0000000000000..96ec25285133c --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/label_encoder_transformer.cc @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/LabelEncoderFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct LabelEncoderTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::LabelEncoderTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::LabelEncoderTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + std::uint32_t* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + } +}; + +class LabelEncoderTransformer final : public OpKernel { + public: + explicit LabelEncoderTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + LabelEncoderTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + LabelEncoderTransformer); +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc b/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc index 98b7845997e34..8fea4f9fb806e 100644 --- a/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/max_abs_scalar_transformer.cc @@ -3,6 +3,7 @@ #include "core/common/common.h" #include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" #include "core/framework/op_kernel.h" #include "Featurizers/MaxAbsScalarFeaturizer.h" @@ -35,12 +36,8 @@ template <> struct OutputTypeMapper { using type = double; }; template -class MaxAbsScalarTransformer final : public OpKernel { - public: - explicit MaxAbsScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { +struct MaxAbsScalarTransformerImpl { + void operator()(OpKernelContext* ctx) const { // Create the transformer Microsoft::Featurizer::Featurizers::MaxAbsScalarTransformer::type> transformer( [ctx](void) { @@ -65,110 +62,39 @@ class MaxAbsScalarTransformer final : public OpKernel { for (int64_t i = 0; i < length; ++i) { output_data[i] = transformer.execute(input_data[i]); } - - return Status::OK(); } }; -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - float, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - int64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); +class MaxAbsScalarTransformer final : public OpKernel { + public: + explicit MaxAbsScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { + } -ONNX_OPERATOR_TYPED_KERNEL_EX( - MaxAbsScalarTransformer, - kMSFeaturizersDomain, - 1, - uint64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; -ONNX_OPERATOR_TYPED_KERNEL_EX( +ONNX_OPERATOR_KERNEL_EX( MaxAbsScalarTransformer, kMSFeaturizersDomain, 1, - double, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - MaxAbsScalarTransformer); - + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + MaxAbsScalarTransformer); } // namespace featurizers } // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/min_max_scalar_transformer.cc b/onnxruntime/featurizers_ops/cpu/min_max_scalar_transformer.cc new file mode 100644 index 0000000000000..281e9cd8099d2 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/min_max_scalar_transformer.cc @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/MinMaxScalarFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct MinMaxScalarTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::MinMaxScalarTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::MinMaxScalarTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + double* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + } +}; + +class MinMaxScalarTransformer final : public OpKernel { + public: + explicit MinMaxScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + MinMaxScalarTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + MinMaxScalarTransformer); +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/missing_dummies_transformer.cc b/onnxruntime/featurizers_ops/cpu/missing_dummies_transformer.cc new file mode 100644 index 0000000000000..255bfd78d3cc3 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/missing_dummies_transformer.cc @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/MissingDummiesFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +inline float const& PreprocessOptional(float const& value) { return value; } +inline double const& PreprocessOptional(double const& value) { return value; } +inline nonstd::optional PreprocessOptional(std::string value) { + return value.empty() ? nonstd::optional() : nonstd::optional(std::move(value)); +} + +template +struct MissingDummiesTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::MissingDummiesTransformer transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::MissingDummiesTransformer(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + int8_t* output_data(output_tensor->MutableData()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(PreprocessOptional(input_data[i])); + } + } +}; + +class MissingDummiesTransformer final : public OpKernel { + public: + explicit MissingDummiesTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + MissingDummiesTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + MissingDummiesTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/robust_scalar_transformer.cc b/onnxruntime/featurizers_ops/cpu/robust_scalar_transformer.cc new file mode 100644 index 0000000000000..728c9c710508f --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/robust_scalar_transformer.cc @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/RobustScalarFeaturizer.h" +#include "Archive.h" + +namespace onnxruntime { +namespace featurizers { + +template +struct OutputTypeMapper {}; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = float; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; +template <> +struct OutputTypeMapper { using type = double; }; + +template +struct RobustScalarTransformerImpl { + void operator()(OpKernelContext* ctx) const { + // Create the transformer + Microsoft::Featurizer::Featurizers::RobustScalarTransformer::type> transformer( + [ctx](void) { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::RobustScalarTransformer::type>(archive); + }()); + + // Get the input + const auto* input_tensor(ctx->Input(1)); + const InputT* input_data(input_tensor->Data()); + + // Prepare the output + Tensor* output_tensor(ctx->Output(0, input_tensor->Shape())); + typename OutputTypeMapper::type* output_data(output_tensor->MutableData::type>()); + + // Execute + const int64_t length(input_tensor->Shape().Size()); + + for (int64_t i = 0; i < length; ++i) { + output_data[i] = transformer.execute(input_data[i]); + } + } +}; + +class RobustScalarTransformer final : public OpKernel { + public: + explicit RobustScalarTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + RobustScalarTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + RobustScalarTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/string_transformer.cc b/onnxruntime/featurizers_ops/cpu/string_transformer.cc index 8f719552dd309..6ee65b8e2633d 100644 --- a/onnxruntime/featurizers_ops/cpu/string_transformer.cc +++ b/onnxruntime/featurizers_ops/cpu/string_transformer.cc @@ -3,6 +3,7 @@ #include "core/common/common.h" #include "core/framework/data_types.h" +#include "core/framework/data_types_internal.h" #include "core/framework/op_kernel.h" #include "Featurizers/StringFeaturizer.h" @@ -12,12 +13,8 @@ namespace onnxruntime { namespace featurizers { template -class StringTransformer final : public OpKernel { - public: - explicit StringTransformer(const OpKernelInfo& info) : OpKernel(info) { - } - - Status Compute(OpKernelContext* ctx) const override { +struct StringTransformerImpl { + void operator()(OpKernelContext* ctx) const { // Create the transformer Microsoft::Featurizer::Featurizers::StringTransformer transformer( [ctx](void) { @@ -42,130 +39,43 @@ class StringTransformer final : public OpKernel { for (int64_t i = 0; i < length; ++i) { output_data[i] = transformer.execute(input_data[i]); } - - return Status::OK(); } }; -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - int64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint8, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint16, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint32, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - uint64, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - float, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); - -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - double, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); +class StringTransformer final : public OpKernel { + public: + explicit StringTransformer(const OpKernelInfo& info) : OpKernel(info) { + } -ONNX_OPERATOR_TYPED_KERNEL_EX( - StringTransformer, - kMSFeaturizersDomain, - 1, - bool, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); + Status Compute(OpKernelContext* ctx) const override { + utils::MLTypeCallDispatcher + t_disp(ctx->Input(1)->GetElementType()); + t_disp.Invoke(ctx); + return Status::OK(); + } +}; -ONNX_OPERATOR_TYPED_KERNEL_EX( +ONNX_OPERATOR_KERNEL_EX( StringTransformer, kMSFeaturizersDomain, 1, - string, kCpuExecutionProvider, KernelDefBuilder() - .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), - StringTransformer); + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), + StringTransformer); } // namespace featurizers } // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc index 4acd5432ec22f..5f246c894b2b9 100644 --- a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc +++ b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc @@ -10,61 +10,28 @@ namespace onnxruntime { namespace featurizers { // Forward declarations -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float, CatImputerTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double, CatImputerTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, CatImputerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CatImputerTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double, MaxAbsScalarTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int8, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int16, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int32, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, int64, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint8, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint16, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint32, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, uint64, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, float, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, double, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, bool, StringTransformer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, string, StringTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, ImputationMarkerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, LabelEncoderTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MaxAbsScalarTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MinMaxScalarTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, MissingDummiesTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, RobustScalarTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StringTransformer); Status RegisterCpuMSFeaturizersKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo}; + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + }; for (auto& function_table_entry : function_table) { ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); diff --git a/onnxruntime/test/featurizers_ops/categoryimputer_test.cc b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc index d803b06879cce..5dc50cc245de2 100644 --- a/onnxruntime/test/featurizers_ops/categoryimputer_test.cc +++ b/onnxruntime/test/featurizers_ops/categoryimputer_test.cc @@ -11,7 +11,7 @@ namespace dft = Microsoft::Featurizer::Featurizers; namespace onnxruntime { namespace test { -TEST(CategoryImputer, Float_values) { +TEST(FeaturizersTests, CategoryImputer_float_values) { OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); @@ -20,15 +20,15 @@ TEST(CategoryImputer, Float_values) { test.AddInput("State", {8}, {1, 0, 0, 0, 0, 0, 192, 63}); // We are adding a scalar Tensor in this instance - test.AddInput("Input", {5}, {1, std::nanf("1"), std::nanf("1"), 2, std::nanf("1")}); + test.AddInput("Input", {5}, {1.f, std::nanf("1"), std::nanf("1"), 2.f, std::nanf("1")}); // Expected output. - test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + test.AddOutput("Output", {5}, {1.f, 1.5f, 1.5f, 2.f, 1.5f}); test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(CategoryImputer, Double_values) { +TEST(FeaturizersTests, CategoryImputer_double_values) { OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State from when the transformer was trained. Corresponds to a @@ -36,15 +36,15 @@ TEST(CategoryImputer, Double_values) { test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 248, 63}); // We are adding a scalar Tensor in this instance - test.AddInput("Input", {5}, {1, std::nan("1"), std::nan("1"), 2, std::nan("1")}); + test.AddInput("Input", {5}, {1., std::nan("1"), std::nan("1"), 2., std::nan("1")}); // Expected output. - test.AddOutput("Output", {5}, {1, 1.5, 1.5, 2, 1.5}); + test.AddOutput("Output", {5}, {1., 1.5, 1.5, 2., 1.5}); test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(CategoryImputer, String_values) { +TEST(FeaturizersTests, CategoryImputer_string_values) { OpTester test("CatImputerTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State from when the transformer was trained. Corresponds to a diff --git a/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc index 863fa7f1a2872..1044667fd54be 100644 --- a/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc +++ b/onnxruntime/test/featurizers_ops/datetimetransformer_test.cc @@ -13,7 +13,7 @@ using SysClock = std::chrono::system_clock; namespace onnxruntime { namespace test { -TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { +TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_04) { const time_t date = 217081624; OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); @@ -74,7 +74,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_04) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05) { +TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17_12_27_05) { const time_t date = 217081625; OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); @@ -135,7 +135,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27_04) { +TEST(FeaturizersTests, DateTimeTransformer_past_1976_nov_17__12_27_05_and_past_1976_nov_17_12_27_04) { const time_t date1 = 217081625; const time_t date2 = 217081624; @@ -223,7 +223,7 @@ TEST(DateTimeTransformer, Past_1976_Nov_17__12_27_05_and_Past_1976_Nov_17__12_27 test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(DateTimeTransformer, Future_2025_June_30) { +TEST(FeaturizersTests, DateTimeTransformer_future_2025_june_30) { const time_t date = 1751241600; OpTester test("DateTimeTransformer", 1, onnxruntime::kMSFeaturizersDomain); diff --git a/onnxruntime/test/featurizers_ops/imputationmarkertransformer_test.cc b/onnxruntime/test/featurizers_ops/imputationmarkertransformer_test.cc new file mode 100644 index 0000000000000..df83269ead6d3 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/imputationmarkertransformer_test.cc @@ -0,0 +1,146 @@ +// ---------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License +// ---------------------------------------------------------------------- + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/ImputationMarkerFeaturizer.h" + +namespace ft = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +std::vector GetStream () { + ft::Archive ar; + ft::Featurizers::ImputationMarkerTransformer inst; + inst.save(ar); + return ar.commit(); +} + +//TEST (FeaturizersTests, ImputationMarker_int8) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint8) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_int16) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint16) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_int32) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint32) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_int64) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} +// +//TEST(FeaturizersTests, ImputationMarker_uint64) { +// OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); +// auto stream = GetStream(); +// auto dim = static_cast(stream.size()); +// +// test.AddInput("State", {dim}, stream); +// test.AddInput("Input", {1}, {25}); +// test.AddOutput("Output", {1}, {false}); +// test.Run(OpTester::ExpectResult::kExpectSuccess); +//} + +TEST(FeaturizersTests, ImputationMarker_float) { + OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5f, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {false, true}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, ImputationMarker_double) { + OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {false, true}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, ImputationMarker_string) { + OpTester test("ImputationMarkerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {"hello", ""}); + test.AddOutput("Output", {2}, {false, true}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +} +} diff --git a/onnxruntime/test/featurizers_ops/labelencodertransfomer_test.cc b/onnxruntime/test/featurizers_ops/labelencodertransfomer_test.cc new file mode 100644 index 0000000000000..32f1c380a6744 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/labelencodertransfomer_test.cc @@ -0,0 +1,110 @@ +// ---------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License +// ---------------------------------------------------------------------- + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Archive.h" +#include "Featurizers/LabelEncoderFeaturizer.h" +#include "Featurizers/TestHelpers.h" + +namespace ft = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +using IndexMap = std::unordered_map; + +template +std::vector GetStream(const IndexMap& map, bool allowMissingValues) { + ft::Archive ar; + using TransType = ft::Featurizers::LabelEncoderTransformer; + TransType inst(map, allowMissingValues); + inst.save(ar); + return ar.commit(); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_uint32) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = uint32_t; + + IndexMap index_map = { + {11, 2}, {8, 0}, {10, 1}, {15, 3}, {20, 5}}; + + auto stream = GetStream(index_map, false); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {5}, {11, 8, 10, 15, 20}); + test.AddOutput("Output", {5}, {2, 0, 1, 3, 5}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_string) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = std::string; + + IndexMap index_map = { + {"orange", 5}, {"apple", 0}, {"grape", 3}, {"carrot", 5}, {"peach", 5}, {"banana", 1}}; + + auto stream = GetStream(index_map, false); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {3}, {"banana", "grape", "apple"}); + test.AddOutput("Output", {3}, {1, 3, 0}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_string_nothrow) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = std::string; + + // when an inference data is not seen before, in the non-throw mode, the featurizer should generate 0 + // hello is not seen before among fruits + IndexMap index_map = { + {"banana", 1}, + {"apple", 2}, + {"grape", 3}, + {"carrot", 4}, + {"peach", 5}, + {"orange", 6}}; + + auto stream = GetStream(index_map, true); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {3}, {"banana", "grape", "hello"}); + // The transformer will add 1 to each of the output for the missing input + test.AddOutput("Output", {3}, {2, 4, 0}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, LabelEncodeTransformer_string_throw) { + OpTester test("LabelEncoderTransformer", 1, onnxruntime::kMSFeaturizersDomain); + using InputType = std::string; + + // when an inference data is not seen before, in the non-throw mode, the featurizer should generate 0 + // hello is not seen before among fruits + IndexMap index_map = { + {"banana", 1}, + {"apple", 2}, + {"grape", 3}, + {"carrot", 4}, + {"peach", 5}, + {"orange", 6}}; + + auto stream = GetStream(index_map, false); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {4}, {"banana", "grape", "apple", "hello"}); + test.AddOutput("Output", {4}, {1, 3, 2, 0}); + test.Run(OpTester::ExpectResult::kExpectFailure, "'input' was not found"); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc b/onnxruntime/test/featurizers_ops/maxabsscalertransformer_test.cc similarity index 79% rename from onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc rename to onnxruntime/test/featurizers_ops/maxabsscalertransformer_test.cc index a871eec3efe26..3bc607f5289fa 100644 --- a/onnxruntime/test/featurizers_ops/maxabsscalerfeaturizer_test.cc +++ b/onnxruntime/test/featurizers_ops/maxabsscalertransformer_test.cc @@ -11,7 +11,7 @@ namespace dft = Microsoft::Featurizer::Featurizers; namespace onnxruntime { namespace test { -TEST(MaxAbsScaler, Int8_values) { +TEST(FeaturizersTests, MaxAbsScaler_int8_values) { OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); @@ -23,12 +23,12 @@ TEST(MaxAbsScaler, Int8_values) { test.AddInput("X", {5}, {-4,3,0,2,-1}); // Expected output. - test.AddOutput("ScaledValues", {5}, {-1,.75,0,.5,-.25}); + test.AddOutput("ScaledValues", {5}, {-1.f,.75f,0.f,.5f,-.25f}); test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(MaxAbsScaler, Double_values) { +TEST(FeaturizersTests, MaxAbsScaler_double_values) { OpTester test("MaxAbsScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State from when the transformer was trained. Corresponds to Version 1 and a @@ -36,10 +36,10 @@ TEST(MaxAbsScaler, Double_values) { test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 64}); // We are adding a scalar Tensor in this instance - test.AddInput("X", {5}, {-4, 3, 0, 2, -1}); + test.AddInput("X", {5}, {-4, 3, 0, 2, -1}); // Expected output. - test.AddOutput("ScaledValues", {5}, {-1, .75, 0, .5, -.25}); + test.AddOutput("ScaledValues", {5}, {-1, .75, 0, .5, -.25}); test.Run(OpTester::ExpectResult::kExpectSuccess); } diff --git a/onnxruntime/test/featurizers_ops/minmaxscalartransformer_test.cc b/onnxruntime/test/featurizers_ops/minmaxscalartransformer_test.cc new file mode 100644 index 0000000000000..4551616b5d83f --- /dev/null +++ b/onnxruntime/test/featurizers_ops/minmaxscalartransformer_test.cc @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/MinMaxScalarFeaturizer.h" + +namespace onnxruntime { +namespace test { + +TEST(FeaturizersTests, MinMaxScalarTransformer_int8) { + OpTester test("MinMaxScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {6}, {1, 0, 0, 0, 1, 9}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {1}, {15}); + + // Expected output. + test.AddOutput("?2", {1}, {1.75}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, MinMaxScalarTransformer_float_t) { + OpTester test("MinMaxScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 128, 191, 0, 0, 128, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {1}, {2.f}); + + // Expected output. + test.AddOutput("?2", {1}, {1.5}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, MinMaxScalarTransformer_only_one_input) { + OpTester test("MinMaxScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {6}, {1, 0, 0, 0, 255, 255}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {1}, {2}); + + // Expected output. + test.AddOutput("?2", {1}, {0}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +} +} \ No newline at end of file diff --git a/onnxruntime/test/featurizers_ops/missingdummiestransfomer_test.cc b/onnxruntime/test/featurizers_ops/missingdummiestransfomer_test.cc new file mode 100644 index 0000000000000..a457b4a5f4f43 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/missingdummiestransfomer_test.cc @@ -0,0 +1,58 @@ +// ---------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License +// ---------------------------------------------------------------------- + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Archive.h" +#include "Featurizers/MissingDummiesFeaturizer.h" + +namespace ft = Microsoft::Featurizer; + +namespace onnxruntime { +namespace test { + +template +std::vector GetStream() { + ft::Archive ar; + ft::Featurizers::MissingDummiesTransformer inst; + inst.save(ar); + return ar.commit(); +} + +TEST(FeaturizersTests, MissingDummiesTransformer_float) { + OpTester test("MissingDummiesTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5f, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {0, 1}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, MissingDummiesTransformer_double) { + OpTester test("MissingDummiesTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {2.5, std::numeric_limits::quiet_NaN()}); + test.AddOutput("Output", {2}, {0, 1}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + +TEST(FeaturizersTests, MissingDummiesTransformer_string) { + OpTester test("MissingDummiesTransformer", 1, onnxruntime::kMSFeaturizersDomain); + auto stream = GetStream(); + auto dim = static_cast(stream.size()); + + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {2}, {"hello", ""}); + test.AddOutput("Output", {2}, {0, 1}); + test.Run(OpTester::ExpectResult::kExpectSuccess); +} +} +} diff --git a/onnxruntime/test/featurizers_ops/robustscalartransformer_test.cc b/onnxruntime/test/featurizers_ops/robustscalartransformer_test.cc new file mode 100644 index 0000000000000..60f809324e95b --- /dev/null +++ b/onnxruntime/test/featurizers_ops/robustscalartransformer_test.cc @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/RobustScalarFeaturizer.h" + +namespace onnxruntime { +namespace test { + +TEST(FeaturizersTests, RobustScalarTransformer_default_with_centering) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 160, 64, 0, 0, 128, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {-1.0f,-0.5f, 0.0f, 0.5f, 1.0f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_no_centering) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 64}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {0.25f, 0.75f, 1.25f, 1.75f, 2.25f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_no_centering_zero_scale) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {3}, {10, 10, 10}); + + // Expected output. + test.AddOutput("?2", {3}, {10.f, 10.f, 10.f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_with_centering_no_scaling) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 160, 64, 0, 0, 128, 63}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {-4.f, -2.f, 0.f, 2.f, 4.f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_with_centering_custom_scaling) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 160, 64, 0, 0, 0, 65}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {-0.5f, -0.25f, 0.f, 0.25f, 0.5f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +TEST(FeaturizersTests, RobustScalarTransformer_default_no_centering_custom_scaling) { + OpTester test("RobustScalarTransformer", 1, onnxruntime::kMSFeaturizersDomain); + + // Add state input + test.AddInput("State", {12}, {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65}); + + // We are adding a scalar Tensor in this instance + test.AddInput("?1", {5}, {1, 3, 5, 7, 9}); + + // Expected output. + test.AddOutput("?2", {5}, {0.125f, 0.375f, 0.625f, 0.875f, 1.125f}); + + test.Run(OpTester::ExpectResult::kExpectSuccess); +} + + +} +} diff --git a/onnxruntime/test/featurizers_ops/stringtransformer_test.cc b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc index c2564d15c73e4..40af22313b9f7 100644 --- a/onnxruntime/test/featurizers_ops/stringtransformer_test.cc +++ b/onnxruntime/test/featurizers_ops/stringtransformer_test.cc @@ -11,7 +11,7 @@ namespace dft = Microsoft::Featurizer::Featurizers; namespace onnxruntime { namespace test { -TEST(StringTransformer, Integer_values) { +TEST(FeaturizersTests, StringTransformer_integer_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1 @@ -26,7 +26,7 @@ TEST(StringTransformer, Integer_values) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(StringTransformer, Double_values) { +TEST(FeaturizersTests, StringTransformer_double_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1 @@ -41,7 +41,7 @@ TEST(StringTransformer, Double_values) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(StringTransformer, Bool_values) { +TEST(FeaturizersTests, StringTransformer_bool_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1 @@ -56,7 +56,7 @@ TEST(StringTransformer, Bool_values) { test.Run(OpTester::ExpectResult::kExpectSuccess); } -TEST(StringTransformer, String_values) { +TEST(FeaturizersTests, StringTransformer_string_values) { OpTester test("StringTransformer", 1, onnxruntime::kMSFeaturizersDomain); // State represents version 1