From a7f96353cd8dbffc4cfc5ad614e6324ba5204129 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Wed, 5 Feb 2020 15:51:08 -0800 Subject: [PATCH 1/6] needs casting to chars --- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 7fd42545fb..3c4e7554c1 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -921,6 +921,38 @@ public void WordEmbeddingsTest() Done(); } + [Fact] + public void TokenizingByCharactersOnnxConversionTest() + { + var mlContext = new MLContext(seed: 1); + var dataPath = GetDataPath("wikipedia-detox-250-line-test.tsv"); + var dataView = ML.Data.LoadFromTextFile(dataPath, new[] { + new TextLoader.Column("label", DataKind.Boolean, 0), + new TextLoader.Column("text", DataKind.String, 1) + }, hasHeader: true); + var pipeline = new TokenizingByCharactersEstimator(mlContext, columns: new[] { ("TokenizedText", "text") }); + var model = pipeline.Fit(dataView); + var transformedData = model.Transform(dataView); + var onnxModel = mlContext.Model.ConvertToOnnxProtobuf(model, dataView); + //CompareSelectedVectorColumns(transformedData.Schema[2].Name, transformedData.Schema[2].Name, transformedData, transformedData); + // Compare model scores produced by ML.NET and ONNX's runtime. + if (IsOnnxRuntimeSupported()) + { + var onnxFileName = $"TokenizingByCharacters.onnx"; + var onnxModelPath = GetOutputPath(onnxFileName); + SaveOnnxModel(onnxModel, onnxModelPath, null); + // Evaluate the saved ONNX model using the data used to train the ML.NET pipeline. + string[] inputNames = onnxModel.Graph.Input.Select(valueInfoProto => valueInfoProto.Name).ToArray(); + string[] outputNames = onnxModel.Graph.Output.Select(valueInfoProto => valueInfoProto.Name).ToArray(); + var onnxEstimator = mlContext.Transforms.ApplyOnnxModel(outputNames, inputNames, onnxModelPath); + var onnxTransformer = onnxEstimator.Fit(dataView); + var onnxResult = onnxTransformer.Transform(dataView); + CompareSelectedVectorColumns(transformedData.Schema[2].Name, outputNames[2], transformedData, onnxResult); //compare scores + + } + Done(); + } + [Theory] // These are the supported conversions // ML.NET does not allow any conversions between signed and unsigned numeric types From 1eb72a8638c2fc1774a39e2e97bfc6564cf3a90e Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Wed, 5 Feb 2020 15:55:39 -0800 Subject: [PATCH 2/6] needs casting to chars --- .../Text/TokenizingByCharacters.cs | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs index bb403d5577..9ac6cccf6b 100644 --- a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs +++ b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs @@ -11,6 +11,7 @@ using Microsoft.ML.CommandLine; using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; +using Microsoft.ML.Model.OnnxConverter; using Microsoft.ML.Runtime; using Microsoft.ML.Transforms.Text; @@ -184,7 +185,7 @@ private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, Dat private protected override IRowMapper MakeRowMapper(DataViewSchema schema) => new Mapper(this, schema); - private sealed class Mapper : OneToOneMapperBase + private sealed class Mapper : OneToOneMapperBase, ISaveAsOnnx { private readonly DataViewType _type; private readonly TokenizingByCharactersTransformer _parent; @@ -204,6 +205,44 @@ public Mapper(TokenizingByCharactersTransformer parent, DataViewSchema inputSche _isSourceVector[i] = inputSchema[_parent.ColumnPairs[i].inputColumnName].Type is VectorDataViewType; } + public bool CanSaveOnnx(OnnxContext ctx) => true; + public void SaveAsOnnx(OnnxContext ctx) + { + Host.CheckValue(ctx, nameof(ctx)); + for (int iinfo = 0; iinfo < _isSourceVector.Length; ++iinfo) + { + string inputColumnName = _parent.ColumnPairs[iinfo].inputColumnName; + if (!ctx.ContainsColumn(inputColumnName)) + continue; + + string outputColumnName = _parent.ColumnPairs[iinfo].outputColumnName; + string srcVariableName = ctx.GetVariableName(inputColumnName); + string dstVariableName = ctx.AddIntermediateVariable(_type, outputColumnName, true); + SaveAsOnnxCore(ctx, srcVariableName, dstVariableName); + } + } + + private void SaveAsOnnxCore(OnnxContext ctx, string srcVariableName, string dstVariableName) + { + string opType = "Tokenizer"; + string tokenizerOutput = ctx.AddIntermediateVariable(null, "TokenizerOutput" , true); + var node = ctx.CreateNode(opType, srcVariableName, tokenizerOutput, ctx.GetNodeName(opType), "com.microsoft"); + node.AddAttribute("mark", _parent._useMarkerChars); + node.AddAttribute("mincharnum", 1); + node.AddAttribute("pad_value", ""); + node.AddAttribute("separators", new string[] { "" }); + + opType = "Squeeze"; + var squeezeOutput = ctx.AddIntermediateVariable(_type, "SqueezeOutput", true); + node = ctx.CreateNode(opType, tokenizerOutput, dstVariableName, ctx.GetNodeName(opType), ""); + node.AddAttribute("axes", new long[] { 0 }); + + //opType = "Cast"; + //node = ctx.CreateNode(opType, squeezeOutput, dstVariableName, ctx.GetNodeName(opType), ""); + //var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.UInt16).ToType(); + //node.AddAttribute("to", t); + + } protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() { var result = new DataViewSchema.DetachedColumn[_parent.ColumnPairs.Length]; From 910edacd1a35fb8cfba77ae4ec04133748e92de8 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Thu, 6 Feb 2020 15:13:25 -0800 Subject: [PATCH 3/6] tokenizer export --- .../Text/TokenizingByCharacters.cs | 26 +++++++++++++------ test/Microsoft.ML.Tests/OnnxConversionTest.cs | 2 -- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs index 9ac6cccf6b..f9fb7101b0 100644 --- a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs +++ b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs @@ -206,6 +206,7 @@ public Mapper(TokenizingByCharactersTransformer parent, DataViewSchema inputSche } public bool CanSaveOnnx(OnnxContext ctx) => true; + public void SaveAsOnnx(OnnxContext ctx) { Host.CheckValue(ctx, nameof(ctx)); @@ -225,7 +226,7 @@ public void SaveAsOnnx(OnnxContext ctx) private void SaveAsOnnxCore(OnnxContext ctx, string srcVariableName, string dstVariableName) { string opType = "Tokenizer"; - string tokenizerOutput = ctx.AddIntermediateVariable(null, "TokenizerOutput" , true); + string tokenizerOutput = ctx.AddIntermediateVariable(null, "TokenizerOutput", true); var node = ctx.CreateNode(opType, srcVariableName, tokenizerOutput, ctx.GetNodeName(opType), "com.microsoft"); node.AddAttribute("mark", _parent._useMarkerChars); node.AddAttribute("mincharnum", 1); @@ -233,16 +234,25 @@ private void SaveAsOnnxCore(OnnxContext ctx, string srcVariableName, string dstV node.AddAttribute("separators", new string[] { "" }); opType = "Squeeze"; - var squeezeOutput = ctx.AddIntermediateVariable(_type, "SqueezeOutput", true); - node = ctx.CreateNode(opType, tokenizerOutput, dstVariableName, ctx.GetNodeName(opType), ""); + var squeezeOutput = ctx.AddIntermediateVariable(null, "SqueezeOutput", true); + node = ctx.CreateNode(opType, tokenizerOutput, squeezeOutput, ctx.GetNodeName(opType), ""); node.AddAttribute("axes", new long[] { 0 }); - //opType = "Cast"; - //node = ctx.CreateNode(opType, squeezeOutput, dstVariableName, ctx.GetNodeName(opType), ""); - //var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.UInt16).ToType(); - //node.AddAttribute("to", t); - + opType = "LabelEncoder"; + var labelEncoderOutput = ctx.AddIntermediateVariable(null, "LabelEncoderOutput", true); + node = ctx.CreateNode(opType, squeezeOutput, labelEncoderOutput, ctx.GetNodeName(opType)); + //GetKeyValues(iinfo, ref dst); + IEnumerable charStrings = Enumerable.Range(0, 65535).Select(x => ((char)x).ToString()); + IEnumerable charValues = Enumerable.Range(0, 65535).Select(x => Convert.ToInt64(x)); ; + node.AddAttribute("keys_strings", charStrings); + node.AddAttribute("values_int64s", charValues); + + opType = "Cast"; + var castNode = ctx.CreateNode(opType, labelEncoderOutput, dstVariableName, ctx.GetNodeName(opType), ""); + var t = InternalDataKindExtensions.ToInternalDataKind(DataKind.UInt16).ToType(); + castNode.AddAttribute("to", t); } + protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() { var result = new DataViewSchema.DetachedColumn[_parent.ColumnPairs.Length]; diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 3c4e7554c1..3e9a591835 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -934,7 +934,6 @@ public void TokenizingByCharactersOnnxConversionTest() var model = pipeline.Fit(dataView); var transformedData = model.Transform(dataView); var onnxModel = mlContext.Model.ConvertToOnnxProtobuf(model, dataView); - //CompareSelectedVectorColumns(transformedData.Schema[2].Name, transformedData.Schema[2].Name, transformedData, transformedData); // Compare model scores produced by ML.NET and ONNX's runtime. if (IsOnnxRuntimeSupported()) { @@ -948,7 +947,6 @@ public void TokenizingByCharactersOnnxConversionTest() var onnxTransformer = onnxEstimator.Fit(dataView); var onnxResult = onnxTransformer.Transform(dataView); CompareSelectedVectorColumns(transformedData.Schema[2].Name, outputNames[2], transformedData, onnxResult); //compare scores - } Done(); } From 05323306729624a34cc53579eac0ee6a993017f7 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Thu, 6 Feb 2020 15:15:59 -0800 Subject: [PATCH 4/6] remove comment --- src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs index f9fb7101b0..f673c4184d 100644 --- a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs +++ b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs @@ -241,7 +241,7 @@ private void SaveAsOnnxCore(OnnxContext ctx, string srcVariableName, string dstV opType = "LabelEncoder"; var labelEncoderOutput = ctx.AddIntermediateVariable(null, "LabelEncoderOutput", true); node = ctx.CreateNode(opType, squeezeOutput, labelEncoderOutput, ctx.GetNodeName(opType)); - //GetKeyValues(iinfo, ref dst); + IEnumerable charStrings = Enumerable.Range(0, 65535).Select(x => ((char)x).ToString()); IEnumerable charValues = Enumerable.Range(0, 65535).Select(x => Convert.ToInt64(x)); ; node.AddAttribute("keys_strings", charStrings); From 774c16f939e9bc8335104489f0f5d68d2e1000ff Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Thu, 6 Feb 2020 15:51:45 -0800 Subject: [PATCH 5/6] resolving comments --- src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs | 2 +- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs index f673c4184d..9aa1772de3 100644 --- a/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs +++ b/src/Microsoft.ML.Transforms/Text/TokenizingByCharacters.cs @@ -607,7 +607,7 @@ private ValueGetter> MakeGetterVec(DataViewRow input, int iinfo) /// | Does this estimator need to look at the data to train its parameters? | Yes | /// | Input column data type | Scalar or Vector of [Text](xref:Microsoft.ML.Data.TextDataViewType) | /// | Output column data type | Variable-sized vector of [key](xref:Microsoft.ML.Data.KeyDataViewType) type. | - /// | Exportable to ONNX | No | + /// | Exportable to ONNX | Yes | /// /// The estimator tokenizes characters by splitting text into sequences of characters using a sliding window. /// During training, the estimator builds a key-value pair dictionary with the encountered sequences of characters. diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 3e9a591835..b7da9dab3f 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -921,8 +921,9 @@ public void WordEmbeddingsTest() Done(); } - [Fact] - public void TokenizingByCharactersOnnxConversionTest() + [Theory] + [CombinatorialData] + public void TokenizingByCharactersOnnxConversionTest(bool useMarkerCharacters) { var mlContext = new MLContext(seed: 1); var dataPath = GetDataPath("wikipedia-detox-250-line-test.tsv"); @@ -930,7 +931,7 @@ public void TokenizingByCharactersOnnxConversionTest() new TextLoader.Column("label", DataKind.Boolean, 0), new TextLoader.Column("text", DataKind.String, 1) }, hasHeader: true); - var pipeline = new TokenizingByCharactersEstimator(mlContext, columns: new[] { ("TokenizedText", "text") }); + var pipeline = new TokenizingByCharactersEstimator(mlContext, useMarkerCharacters: useMarkerCharacters, columns: new[] { ("TokenizedText", "text") }); var model = pipeline.Fit(dataView); var transformedData = model.Transform(dataView); var onnxModel = mlContext.Model.ConvertToOnnxProtobuf(model, dataView); From dab09c8218928d336312ebd56ad5617bd2f4d2e2 Mon Sep 17 00:00:00 2001 From: Keren Fuentes Date: Thu, 6 Feb 2020 16:07:15 -0800 Subject: [PATCH 6/6] updating docs --- .../MulticlassClassification/MulticlassNaiveBayesTrainer.cs | 2 +- src/Microsoft.ML.Transforms/Text/TextNormalizing.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/MulticlassNaiveBayesTrainer.cs b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/MulticlassNaiveBayesTrainer.cs index 20c2d8be37..8bb0371089 100644 --- a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/MulticlassNaiveBayesTrainer.cs +++ b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/MulticlassNaiveBayesTrainer.cs @@ -43,7 +43,7 @@ namespace Microsoft.ML.Trainers /// | Is normalization required? | Yes | /// | Is caching required? | No | /// | Required NuGet in addition to Microsoft.ML | None | - /// | Exportable to ONNX | No | + /// | Exportable to ONNX | Yes | /// /// ### Training Algorithm Details /// [Naive Bayes](https://en.wikipedia.org/wiki/Naive_Bayes_classifier) diff --git a/src/Microsoft.ML.Transforms/Text/TextNormalizing.cs b/src/Microsoft.ML.Transforms/Text/TextNormalizing.cs index 19f26c0ef0..6df8d5a604 100644 --- a/src/Microsoft.ML.Transforms/Text/TextNormalizing.cs +++ b/src/Microsoft.ML.Transforms/Text/TextNormalizing.cs @@ -442,7 +442,7 @@ private bool IsCombiningDiacritic(char ch) /// | Does this estimator need to look at the data to train its parameters? | No | /// | Input column data type | Scalar or Vector of [Text](xref:Microsoft.ML.Data.TextDataViewType)| /// | Output column data type | Scalar or variable-sized Vector of [Text](xref:Microsoft.ML.Data.TextDataViewType)| - /// | Exportable to ONNX | No | + /// | Exportable to ONNX | Yes | /// /// The resulting creates a new column, named as specified /// in the output column name parameters, and normalizes the textual input data by changing case, removing diacritical marks,