From 36695790c4766b2ce7530d2f0e06e08e3c027d9d Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 28 May 2020 01:34:43 -0700 Subject: [PATCH 1/5] Added logic and tests for sanitized column names in CLI --- .../CodeGenerator/CSharp/CodeGenerator.cs | 67 +++++----- .../CodeGenTests.cs | 125 ++++++++++++++---- 2 files changed, 137 insertions(+), 55 deletions(-) diff --git a/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs b/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs index 2a2dbadcc6..603ceb0049 100644 --- a/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs +++ b/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs @@ -260,6 +260,7 @@ private string GenerateConsumeModelCSFileContent(string namespaceValue) internal IList GenerateClassLabels(IDictionary columnMapping = default) { IList result = new List(); + HashSet columnNames = new HashSet(); foreach (var column in _columnInferenceResult.TextLoaderOptions.Columns) { StringBuilder sb = new StringBuilder(); @@ -282,35 +283,7 @@ internal IList GenerateClassLabels(IDictionary 0) { @@ -322,7 +295,16 @@ internal IList GenerateClassLabels(IDictionary GenerateClassLabels(IDictionary inputColumns = new List + { + new TextLoader.Column[] + { + new TextLoader.Column(){ Name = "Label", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.Boolean }, + }, + new TextLoader.Column[] + { + new TextLoader.Column(){ Name = "id", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.Single }, + new TextLoader.Column(){ Name = "country", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.Single }, + new TextLoader.Column(){ Name = "Country", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.String } + }, + new TextLoader.Column[] + { + new TextLoader.Column(){ Name = "vin", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "Make", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "Model", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "model", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "color", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "Color", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "MSRP", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.Single }, + new TextLoader.Column(){ Name = "engine size", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.Double }, + new TextLoader.Column(){ Name = "isElectric", Source = new TextLoader.Range[]{new TextLoader.Range(8) }, DataKind = DataKind.Boolean }, + } + }; + public static List> expectedLabels = new List> + { + new List + { + "[ColumnName(\"Label\"), LoadColumn(0)]", + "public bool Label{get; set;}", + "\r\n" + }, + new List + { + "[ColumnName(\"id\"), LoadColumn(0)]", + "public float Id{get; set;}", + "\r\n", + "[ColumnName(\"country\"), LoadColumn(1)]", + "public float Country{get; set;}", + "\r\n", + "[ColumnName(\"Country\"), LoadColumn(2)]", + "public string Country_string{get; set;}", + "\r\n" + }, + new List + { + "[ColumnName(\"vin\"), LoadColumn(0)]", + "public int Vin{get; set;}", + "\r\n", + "[ColumnName(\"Make\"), LoadColumn(1)]", + "public string Make{get; set;}", + "\r\n", + "[ColumnName(\"Model\"), LoadColumn(2)]", + "public int Model{get; set;}", + "\r\n", + "[ColumnName(\"model\"), LoadColumn(3)]", + "public string Model_string{get; set;}", + "\r\n", + "[ColumnName(\"color\"), LoadColumn(4)]", + "public string Color{get; set;}", + "\r\n", + "[ColumnName(\"Color\"), LoadColumn(5)]", + "public int Color_int{get; set;}", + "\r\n", + "[ColumnName(\"MSRP\"), LoadColumn(6)]", + "public float MSRP{get; set;}", + "\r\n", + "[ColumnName(\"engine size\"), LoadColumn(7)]", + "public double Engine_size{get; set;}", + "\r\n", + "[ColumnName(\"isElectric\"), LoadColumn(8)]", + "public bool IsElectric{get; set;}", + "\r\n" + + } + }; + } } From d476c551a72eae72f4781c14fde43266bbab7fad Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Fri, 29 May 2020 00:00:54 -0500 Subject: [PATCH 2/5] Compressed GenerateClassLabel logic, added new unit tests --- .../CodeGenerator/CSharp/CodeGenerator.cs | 80 +---------- src/Microsoft.ML.CodeGenerator/Utils.cs | 67 ++++----- ...InputClassTest.sanitized_map.approved.txt} | 0 ...ClassTest.sanitized_null_map.approved.txt} | 0 ...nputClassTest.unsanitized_map.approved.txt | 46 +++++++ ...lassTest.unsanitized_null_map.approved.txt | 46 +++++++ .../ConsoleCodeGeneratorTests.cs | 130 +++++++++++++++++- .../CodeGenTests.cs | 26 ++++ 8 files changed, 285 insertions(+), 110 deletions(-) rename test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/{ConsoleCodeGeneratorTests.ModelInputClassTest.map.approved.txt => ConsoleCodeGeneratorTests.ModelInputClassTest.sanitized_map.approved.txt} (100%) rename test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/{ConsoleCodeGeneratorTests.ModelInputClassTest.null_map.approved.txt => ConsoleCodeGeneratorTests.ModelInputClassTest.sanitized_null_map.approved.txt} (100%) create mode 100644 test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt create mode 100644 test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt diff --git a/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs b/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs index 603ceb0049..db57a92c10 100644 --- a/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs +++ b/src/Microsoft.ML.CodeGenerator/CodeGenerator/CSharp/CodeGenerator.cs @@ -257,84 +257,12 @@ private string GenerateConsumeModelCSFileContent(string namespaceValue) return (trainerString, trainerUsings); } + /// + /// Utilize + /// internal IList GenerateClassLabels(IDictionary columnMapping = default) { - IList result = new List(); - HashSet columnNames = new HashSet(); - foreach (var column in _columnInferenceResult.TextLoaderOptions.Columns) - { - StringBuilder sb = new StringBuilder(); - int range = (column.Source[0].Max - column.Source[0].Min).Value; - bool isArray = range > 0; - sb.Append(Symbols.PublicSymbol); - sb.Append(Symbols.Space); - - // if column is in columnMapping, use the type and name in that - DataKind dataKind; - string columnName; - - if (columnMapping != null && columnMapping.ContainsKey(column.Name)) - { - dataKind = columnMapping[column.Name].ColumnType; - columnName = columnMapping[column.Name].ColumnName; - } - else - { - dataKind = column.DataKind; - columnName = column.Name; - } - sb.Append(GetSymbolOfDataKind(dataKind)); - - if (range > 0) - { - result.Add($"[ColumnName(\"{columnName}\"),LoadColumn({column.Source[0].Min}, {column.Source[0].Max}) VectorType({(range + 1)})]"); - sb.Append("[]"); - } - else - { - result.Add($"[ColumnName(\"{columnName}\"), LoadColumn({column.Source[0].Min})]"); - } - sb.Append(" "); - string normalizedColumnName = Utils.Normalize(column.Name); - if (columnNames.Contains(Utils.Normalize(column.Name))) - { - normalizedColumnName = normalizedColumnName + "_" + GetSymbolOfDataKind(dataKind); - if (columnNames.Contains(normalizedColumnName)) - throw new ArgumentException($"The column '{column.Name}' with type '{dataKind}' is not unique in the dataset."); - } - else - columnNames.Add(normalizedColumnName); - sb.Append(normalizedColumnName); - sb.Append("{get; set;}"); - result.Add(sb.ToString()); - result.Add("\r\n"); - } - return result; - } - - public string GetSymbolOfDataKind(DataKind dataKind) - { - switch (dataKind) - { - case DataKind.String: - return Symbols.StringSymbol; - case DataKind.Boolean: - return Symbols.BoolSymbol; - case DataKind.Single: - return Symbols.FloatSymbol; - case DataKind.Double: - return Symbols.DoubleSymbol; - case DataKind.Int32: - return Symbols.IntSymbol; - case DataKind.UInt32: - return Symbols.UIntSymbol; - case DataKind.Int64: - return Symbols.LongSymbol; - case DataKind.UInt64: - return Symbols.UlongSymbol; - default: - throw new ArgumentException($"The data type '{dataKind}' is not handled currently."); - } + return Utils.GenerateClassLabels(_columnInferenceResult, columnMapping); } #region Model project diff --git a/src/Microsoft.ML.CodeGenerator/Utils.cs b/src/Microsoft.ML.CodeGenerator/Utils.cs index 075bc577ce..2435d6f498 100644 --- a/src/Microsoft.ML.CodeGenerator/Utils.cs +++ b/src/Microsoft.ML.CodeGenerator/Utils.cs @@ -246,6 +246,7 @@ internal static int CreateSolutionFile(string solutionFile, string outputPath) internal static IList GenerateClassLabels(ColumnInferenceResults columnInferenceResults, IDictionary columnMapping = default) { IList result = new List(); + HashSet columnNames = new HashSet(); foreach (var column in columnInferenceResults.TextLoaderOptions.Columns) { StringBuilder sb = new StringBuilder(); @@ -268,36 +269,7 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI dataKind = column.DataKind; columnName = column.Name; } - switch (dataKind) - { - case Microsoft.ML.Data.DataKind.String: - sb.Append(Symbols.StringSymbol); - break; - case Microsoft.ML.Data.DataKind.Boolean: - sb.Append(Symbols.BoolSymbol); - break; - case Microsoft.ML.Data.DataKind.Single: - sb.Append(Symbols.FloatSymbol); - break; - case Microsoft.ML.Data.DataKind.Double: - sb.Append(Symbols.DoubleSymbol); - break; - case Microsoft.ML.Data.DataKind.Int32: - sb.Append(Symbols.IntSymbol); - break; - case Microsoft.ML.Data.DataKind.UInt32: - sb.Append(Symbols.UIntSymbol); - break; - case Microsoft.ML.Data.DataKind.Int64: - sb.Append(Symbols.LongSymbol); - break; - case Microsoft.ML.Data.DataKind.UInt64: - sb.Append(Symbols.UlongSymbol); - break; - default: - throw new ArgumentException($"The data type '{column.DataKind}' is not handled currently."); - - } + sb.Append(GetSymbolOfDataKind(dataKind)); if (range > 0) { @@ -309,12 +281,45 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI result.Add($"[ColumnName(\"{columnName}\"), LoadColumn({column.Source[0].Min})]"); } sb.Append(" "); - sb.Append(Utils.Normalize(column.Name)); + string normalizedColumnName = Utils.Normalize(column.Name); + if (columnNames.Contains(normalizedColumnName)) + { + normalizedColumnName = normalizedColumnName + "_" + GetSymbolOfDataKind(dataKind); + if (columnNames.Contains(normalizedColumnName)) + throw new ArgumentException($"The column '{column.Name}' with type '{dataKind}' is not unique in the dataset."); + } + columnNames.Add(normalizedColumnName); + sb.Append(normalizedColumnName); sb.Append("{get; set;}"); result.Add(sb.ToString()); result.Add("\r\n"); } return result; } + + internal static string GetSymbolOfDataKind(DataKind dataKind) + { + switch (dataKind) + { + case DataKind.String: + return Symbols.StringSymbol; + case DataKind.Boolean: + return Symbols.BoolSymbol; + case DataKind.Single: + return Symbols.FloatSymbol; + case DataKind.Double: + return Symbols.DoubleSymbol; + case DataKind.Int32: + return Symbols.IntSymbol; + case DataKind.UInt32: + return Symbols.UIntSymbol; + case DataKind.Int64: + return Symbols.LongSymbol; + case DataKind.UInt64: + return Symbols.UlongSymbol; + default: + throw new ArgumentException($"The data type '{dataKind}' is not handled currently."); + } + } } } diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.sanitized_map.approved.txt similarity index 100% rename from test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.map.approved.txt rename to test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.sanitized_map.approved.txt diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.null_map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.sanitized_null_map.approved.txt similarity index 100% rename from test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.null_map.approved.txt rename to test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.sanitized_null_map.approved.txt diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt new file mode 100644 index 0000000000..e41632a08e --- /dev/null +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt @@ -0,0 +1,46 @@ +//***************************************************************************************** +//* * +//* This is an auto-generated file by Microsoft ML.NET CLI (Command-Line Interface) tool. * +//* * +//***************************************************************************************** + +using Microsoft.ML.Data; + +namespace test.Model +{ + public class ModelInput + { + [ColumnName("input_0"), LoadColumn(0)] + public int Id { get; set; } + + + [ColumnName("input_1"), LoadColumn(1)] + public int MsAssetNum { get; set; } + + + [ColumnName("input_2"), LoadColumn(2)] + public string Make { get; set; } + + + [ColumnName("input_3"), LoadColumn(3)] + public string Model { get; set; } + + + [ColumnName("input_4"), LoadColumn(4)] + public double Model_double { get; set; } + + + [ColumnName("input_5"), LoadColumn(5)] + public string Work_category { get; set; } + + + [ColumnName("Work category"), LoadColumn(6)] + public int Work_category_int { get; set; } + + + [ColumnName("input_7"), LoadColumn(7)] + public bool IsDetachable { get; set; } + + + } +} diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt new file mode 100644 index 0000000000..6aa87d8806 --- /dev/null +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt @@ -0,0 +1,46 @@ +//***************************************************************************************** +//* * +//* This is an auto-generated file by Microsoft ML.NET CLI (Command-Line Interface) tool. * +//* * +//***************************************************************************************** + +using Microsoft.ML.Data; + +namespace test.Model +{ + public class ModelInput + { + [ColumnName("id"), LoadColumn(0)] + public int Id { get; set; } + + + [ColumnName("MsAssetNum"), LoadColumn(1)] + public int MsAssetNum { get; set; } + + + [ColumnName("Make"), LoadColumn(2)] + public string Make { get; set; } + + + [ColumnName("Model"), LoadColumn(3)] + public string Model { get; set; } + + + [ColumnName("model"), LoadColumn(4)] + public double Model_double { get; set; } + + + [ColumnName("work category"), LoadColumn(5)] + public string Work_category { get; set; } + + + [ColumnName("Work category"), LoadColumn(6)] + public int Work_category_int { get; set; } + + + [ColumnName("IsDetachable"), LoadColumn(7)] + public bool IsDetachable { get; set; } + + + } +} diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.cs b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.cs index c7539dddd6..7ffa064d64 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.cs +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.cs @@ -267,8 +267,18 @@ public void AzureCodeGeneratorTest() [MethodImpl(MethodImplOptions.NoInlining)] public void ModelInputClassTest() { - (var pipeline, var columnInference, var mapping) = this.GetMockedAzurePipelineAndInference(); + // Test with datasets whose columns are sanitized and not sanitized. The columns of a dataset are considered + // sanitized if the column names are all unique and distinct, irrespective of capitalization. + (var pipelineSanitized, var columnInferenceSanitized, var mappingSanitized) = this.GetMockedAzurePipelineAndInference(); + TestModelInput(pipelineSanitized, columnInferenceSanitized, mappingSanitized, "sanitized"); + (var pipelineUnsatinized, var columnInferenceUnsatinized, var mappingUnsatinized) = this.GetMockedAzurePipelineAndInferenceUnsanitizedColumnNames(); + TestModelInput(pipelineUnsatinized, columnInferenceUnsatinized, mappingUnsatinized, "unsanitized"); + } + + private void TestModelInput(Pipeline pipeline, ColumnInferenceResults columnInference, + IDictionary mapping, string info) + { // test with null map case var columnMappingStringList = Utils.GenerateClassLabels(columnInference); var modelInputProject = new CSharpCodeFile() @@ -281,7 +291,7 @@ public void ModelInputClassTest() }.TransformText(), Name = "ModelInput.cs", }; - NamerFactory.AdditionalInformation = "null_map"; + NamerFactory.AdditionalInformation = info + "_null_map"; Approvals.Verify(modelInputProject.File); // test with map case @@ -296,7 +306,7 @@ public void ModelInputClassTest() }.TransformText(), Name = "ModelInput.cs", }; - NamerFactory.AdditionalInformation = "map"; + NamerFactory.AdditionalInformation = info + "_map"; Approvals.Verify(modelInputProject.File); } @@ -828,6 +838,120 @@ private CodeGenerator PrepareForRecommendationTask() return (bestPipeLine, columnInference, mapping); } + private (Pipeline, ColumnInferenceResults, IDictionary) GetMockedAzurePipelineAndInferenceUnsanitizedColumnNames() + { + // construct pipeline + var onnxPipeLineNode = new PipelineNode(nameof(SpecialTransformer.ApplyOnnxModel), PipelineNodeType.Transform, new[] { "input.1" }, new[] { "output.1" }, + new Dictionary() + { + { "outputColumnNames", "output1" }, + { "inputColumnNames", "input1"}, + }); + var labelMapPipelineNode = new PipelineNode(nameof(SpecialTransformer.LabelMapping), PipelineNodeType.Transform, string.Empty, string.Empty); + var bestPipeLine = new Pipeline(new PipelineNode[] + { + onnxPipeLineNode, + labelMapPipelineNode, + }); + + // construct column inference + var textLoaderArgs = new TextLoader.Options() + { + Columns = new[] { + new TextLoader.Column("id", DataKind.Int32, 0), + new TextLoader.Column("MsAssetNum", DataKind.Int32, 1), + new TextLoader.Column("Make", DataKind.String, 2), + new TextLoader.Column("Model", DataKind.String, 3), + new TextLoader.Column("model", DataKind.Double, 4), + new TextLoader.Column("work category", DataKind.String, 5), + new TextLoader.Column("Work category", DataKind.Int32, 6), + new TextLoader.Column("IsDetachable", DataKind.Boolean, 7), + }, + AllowQuoting = true, + AllowSparse = true, + HasHeader = true, + Separators = new[] { ',' } + }; + + var columnInference = new ColumnInferenceResults() + { + TextLoaderOptions = textLoaderArgs, + ColumnInformation = new ColumnInformation() { LabelColumnName = "Label" } + }; + + // construct columnMapping + // mock columnMapping + var mapping = new Dictionary() + { + { + "id", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_0", + ColumnType = DataKind.Int32, + } + }, + { + "MsAssetNum", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_1", + ColumnType = DataKind.Int32, + } + }, + { + "Make", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_2", + ColumnType = DataKind.String, + } + }, + { + "Model", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_3", + ColumnType = DataKind.String, + } + }, + { + "model", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_4", + ColumnType = DataKind.Double, + } + }, + { + "work category", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_5", + ColumnType = DataKind.String, + } + }, + { + "Work Category", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_6", + ColumnType = DataKind.Int32, + } + }, + { + "IsDetachable", + new CodeGeneratorSettings.ColumnMapping() + { + ColumnName = "input_7", + ColumnType = DataKind.Boolean, + } + } + }; + + return (bestPipeLine, columnInference, mapping); + } + private (Pipeline, ColumnInferenceResults) GetMockedOvaPipelineAndInference() { diff --git a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs index 3ffa34312b..9f4544080a 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs +++ b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs @@ -151,6 +151,14 @@ public class CodeGenTestData new TextLoader.Column(){ Name = "Country", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.String } }, new TextLoader.Column[] + { + new TextLoader.Column(){ Name = "id", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "shape", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "Shape", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "color", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "price", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.Double }, + }, + new TextLoader.Column[] { new TextLoader.Column(){ Name = "vin", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.Int32 }, new TextLoader.Column(){ Name = "Make", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, @@ -184,6 +192,24 @@ public class CodeGenTestData "\r\n" }, new List + { + "[ColumnName(\"id\"), LoadColumn(0)]", + "public int Id{get; set;}", + "\r\n", + "[ColumnName(\"shape\"), LoadColumn(1)]", + "public int Shape{get; set;}", + "\r\n", + "[ColumnName(\"Shape\"), LoadColumn(2)]", + "public string Shape_string{get; set;}", + "\r\n", + "[ColumnName(\"color\"), LoadColumn(3)]", + "public string Color{get; set;}", + "\r\n", + "[ColumnName(\"price\"), LoadColumn(4)]", + "public double Price{get; set;}", + "\r\n" + }, + new List { "[ColumnName(\"vin\"), LoadColumn(0)]", "public int Vin{get; set;}", From d17e73982dc0df7211ad3412d227850b0439917b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 17 Jun 2020 18:50:02 -0700 Subject: [PATCH 3/5] Add differentiator and more unit tests --- src/Microsoft.ML.CodeGenerator/Utils.cs | 14 +++- .../CodeGenTests.cs | 75 +++++++++++++++---- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ML.CodeGenerator/Utils.cs b/src/Microsoft.ML.CodeGenerator/Utils.cs index 2435d6f498..ad527f1c5d 100644 --- a/src/Microsoft.ML.CodeGenerator/Utils.cs +++ b/src/Microsoft.ML.CodeGenerator/Utils.cs @@ -247,6 +247,7 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI { IList result = new List(); HashSet columnNames = new HashSet(); + Dictionary duplicates = new Dictionary(); foreach (var column in columnInferenceResults.TextLoaderOptions.Columns) { StringBuilder sb = new StringBuilder(); @@ -281,12 +282,23 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI result.Add($"[ColumnName(\"{columnName}\"), LoadColumn({column.Source[0].Min})]"); } sb.Append(" "); + // Obtain normalized version of column name string normalizedColumnName = Utils.Normalize(column.Name); + // Check if there's already a variable with the same normalized column name if (columnNames.Contains(normalizedColumnName)) { + // Add first differentiator to column name normalizedColumnName = normalizedColumnName + "_" + GetSymbolOfDataKind(dataKind); + // Check if there's already a variable with the same normalized column name and type if (columnNames.Contains(normalizedColumnName)) - throw new ArgumentException($"The column '{column.Name}' with type '{dataKind}' is not unique in the dataset."); + { + if (duplicates.ContainsKey(normalizedColumnName)) + duplicates[normalizedColumnName] += 1; + else + duplicates.Add(normalizedColumnName, 1); + // Add second differentiator to column name + normalizedColumnName = normalizedColumnName + "_" + duplicates[normalizedColumnName]; + } } columnNames.Add(normalizedColumnName); sb.Append(normalizedColumnName); diff --git a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs index 9f4544080a..0cd6ffb704 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs +++ b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs @@ -162,13 +162,26 @@ public class CodeGenTestData { new TextLoader.Column(){ Name = "vin", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.Int32 }, new TextLoader.Column(){ Name = "Make", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "Model", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "model", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "color", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "Color", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "MSRP", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.Single }, - new TextLoader.Column(){ Name = "engine size", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.Double }, - new TextLoader.Column(){ Name = "isElectric", Source = new TextLoader.Range[]{new TextLoader.Range(8) }, DataKind = DataKind.Boolean }, + new TextLoader.Column(){ Name = "make", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "Model", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "model", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "model", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "color", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "Color", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "MSRP", Source = new TextLoader.Range[]{new TextLoader.Range(8) }, DataKind = DataKind.Single }, + new TextLoader.Column(){ Name = "engine size", Source = new TextLoader.Range[]{new TextLoader.Range(9) }, DataKind = DataKind.Double }, + new TextLoader.Column(){ Name = "isElectric", Source = new TextLoader.Range[]{new TextLoader.Range(10) }, DataKind = DataKind.Boolean }, + }, + new TextLoader.Column[] + { + new TextLoader.Column(){ Name = "var_text", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "var_text", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "var_text", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "var_text", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.String }, } }; public static List> expectedLabels = new List> @@ -217,28 +230,60 @@ public class CodeGenTestData "[ColumnName(\"Make\"), LoadColumn(1)]", "public string Make{get; set;}", "\r\n", - "[ColumnName(\"Model\"), LoadColumn(2)]", + "[ColumnName(\"make\"), LoadColumn(2)]", + "public int Make_int{get; set;}", + "\r\n", + "[ColumnName(\"Model\"), LoadColumn(3)]", "public int Model{get; set;}", "\r\n", - "[ColumnName(\"model\"), LoadColumn(3)]", + "[ColumnName(\"model\"), LoadColumn(4)]", "public string Model_string{get; set;}", "\r\n", - "[ColumnName(\"color\"), LoadColumn(4)]", + "[ColumnName(\"model\"), LoadColumn(5)]", + "public int Model_int{get; set;}", + "\r\n", + "[ColumnName(\"color\"), LoadColumn(6)]", "public string Color{get; set;}", "\r\n", - "[ColumnName(\"Color\"), LoadColumn(5)]", + "[ColumnName(\"Color\"), LoadColumn(7)]", "public int Color_int{get; set;}", "\r\n", - "[ColumnName(\"MSRP\"), LoadColumn(6)]", + "[ColumnName(\"MSRP\"), LoadColumn(8)]", "public float MSRP{get; set;}", "\r\n", - "[ColumnName(\"engine size\"), LoadColumn(7)]", + "[ColumnName(\"engine size\"), LoadColumn(9)]", "public double Engine_size{get; set;}", "\r\n", - "[ColumnName(\"isElectric\"), LoadColumn(8)]", + "[ColumnName(\"isElectric\"), LoadColumn(10)]", "public bool IsElectric{get; set;}", "\r\n" - + }, + new List + { + "[ColumnName(\"var_text\"), LoadColumn(0)]", + "public string Var_text{get; set;}", + "\r\n", + "[ColumnName(\"var_text\"), LoadColumn(1)]", + "public string Var_text_string{get; set;}", + "\r\n", + "[ColumnName(\"var_num\"), LoadColumn(2)]", + "public int Var_num{get; set;}", + "\r\n", + "[ColumnName(\"var_num\"), LoadColumn(3)]", + "public int Var_num_int{get; set;}", + "\r\n", + "[ColumnName(\"var_num\"), LoadColumn(4)]", + "public int Var_num_int_1{get; set;}", + "\r\n", + "[ColumnName(\"var_text\"), LoadColumn(5)]", + "public string Var_text_string_1{get; set;}", + "\r\n", + "[ColumnName(\"var_num\"), LoadColumn(6)]", + "public int Var_num_int_2{get; set;}", + "\r\n", + "[ColumnName(\"var_text\"), LoadColumn(7)]", + "public string Var_text_string_2{get; set;}", + "\r\n", } }; } From 22ad34c9fbf40b0d26772a3081f1c30779fd6fa9 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 18 Jun 2020 18:53:37 -0700 Subject: [PATCH 4/5] Added better column naming support, updated tests --- src/Microsoft.ML.CodeGenerator/Utils.cs | 57 ++++++++++------ ...nputClassTest.unsanitized_map.approved.txt | 4 +- ...lassTest.unsanitized_null_map.approved.txt | 4 +- .../CodeGenTests.cs | 68 +++++++++++++++---- 4 files changed, 96 insertions(+), 37 deletions(-) diff --git a/src/Microsoft.ML.CodeGenerator/Utils.cs b/src/Microsoft.ML.CodeGenerator/Utils.cs index ad527f1c5d..c715a99ead 100644 --- a/src/Microsoft.ML.CodeGenerator/Utils.cs +++ b/src/Microsoft.ML.CodeGenerator/Utils.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Reflection; using System.Text; +using System.Text.RegularExpressions; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.Formatting; @@ -247,7 +248,7 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI { IList result = new List(); HashSet columnNames = new HashSet(); - Dictionary duplicates = new Dictionary(); + Dictionary<(string, DataKind), int> propertyNames = new Dictionary<(string, DataKind), int>(); foreach (var column in columnInferenceResults.TextLoaderOptions.Columns) { StringBuilder sb = new StringBuilder(); @@ -272,6 +273,7 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI } sb.Append(GetSymbolOfDataKind(dataKind)); + // Accomodate VectorType (array) columns if (range > 0) { result.Add($"[ColumnName(\"{columnName}\"),LoadColumn({column.Source[0].Min}, {column.Source[0].Max}) VectorType({(range + 1)})]"); @@ -282,25 +284,9 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI result.Add($"[ColumnName(\"{columnName}\"), LoadColumn({column.Source[0].Min})]"); } sb.Append(" "); - // Obtain normalized version of column name - string normalizedColumnName = Utils.Normalize(column.Name); - // Check if there's already a variable with the same normalized column name - if (columnNames.Contains(normalizedColumnName)) - { - // Add first differentiator to column name - normalizedColumnName = normalizedColumnName + "_" + GetSymbolOfDataKind(dataKind); - // Check if there's already a variable with the same normalized column name and type - if (columnNames.Contains(normalizedColumnName)) - { - if (duplicates.ContainsKey(normalizedColumnName)) - duplicates[normalizedColumnName] += 1; - else - duplicates.Add(normalizedColumnName, 1); - // Add second differentiator to column name - normalizedColumnName = normalizedColumnName + "_" + duplicates[normalizedColumnName]; - } - } - columnNames.Add(normalizedColumnName); + + // Obtain normalized and unique version of column name + string normalizedColumnName = GetNormalizedColumnName(column.Name, dataKind, ref propertyNames); sb.Append(normalizedColumnName); sb.Append("{get; set;}"); result.Add(sb.ToString()); @@ -309,6 +295,37 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI return result; } + internal static string GetNormalizedColumnName(string rawColumnName, DataKind dataKind, ref Dictionary<(string, DataKind), int> propertyNames) + { + // Get normalized column name for correctly typed class property name + string normalizedColumnName = Utils.Normalize(rawColumnName); + // Check if the normalized column name already has type and/or count in name, and remove that substring if it exists. + // These will be added on as necessary below. + // This regex checks for the "_[dataKind]" and "_[dataKind]_[int]" pattern. + Match match = Regex.Match(normalizedColumnName, $@"_{GetSymbolOfDataKind(dataKind)}_*(\d)*?$"); + if (match.Success) + normalizedColumnName = normalizedColumnName.Substring(0, match.Index); + + // Check if there's already a variable with the same normalized column name and type + if (propertyNames.ContainsKey((normalizedColumnName, dataKind))) + { + propertyNames[(normalizedColumnName, dataKind)] += 1; + normalizedColumnName += String.Concat("_", GetSymbolOfDataKind(dataKind), "_", propertyNames[(normalizedColumnName, dataKind)] - 1); + } + // Check if there's already a variable with the same normalized column name but different type + else if (propertyNames.Keys.ToList().Select(t=>t.Item1).Contains(normalizedColumnName)) + { + propertyNames.Add((normalizedColumnName, dataKind), 1); + normalizedColumnName += String.Concat("_", GetSymbolOfDataKind(dataKind), "_", propertyNames[(normalizedColumnName, dataKind)] - 1); + } + // This normalized column name is unique, add it to dictionary + else + { + propertyNames.Add((normalizedColumnName, dataKind), 1); + } + return normalizedColumnName; + } + internal static string GetSymbolOfDataKind(DataKind dataKind) { switch (dataKind) diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt index e41632a08e..a8498455a3 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt @@ -27,7 +27,7 @@ namespace test.Model [ColumnName("input_4"), LoadColumn(4)] - public double Model_double { get; set; } + public double Model_double_0 { get; set; } [ColumnName("input_5"), LoadColumn(5)] @@ -35,7 +35,7 @@ namespace test.Model [ColumnName("Work category"), LoadColumn(6)] - public int Work_category_int { get; set; } + public int Work_category_int_0 { get; set; } [ColumnName("input_7"), LoadColumn(7)] diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt index 6aa87d8806..08e201bba3 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt @@ -27,7 +27,7 @@ namespace test.Model [ColumnName("model"), LoadColumn(4)] - public double Model_double { get; set; } + public double Model_double_0 { get; set; } [ColumnName("work category"), LoadColumn(5)] @@ -35,7 +35,7 @@ namespace test.Model [ColumnName("Work category"), LoadColumn(6)] - public int Work_category_int { get; set; } + public int Work_category_int_0 { get; set; } [ColumnName("IsDetachable"), LoadColumn(7)] diff --git a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs index 0cd6ffb704..4b8b465374 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs +++ b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs @@ -182,6 +182,18 @@ public class CodeGenTestData new TextLoader.Column(){ Name = "var_text", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.String }, new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.Int32 }, new TextLoader.Column(){ Name = "var_text", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(8) }, DataKind = DataKind.Double }, + }, + new TextLoader.Column[] + { + new TextLoader.Column(){ Name = "column1", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "column1_string", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "column1_string_1", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "column1_string_2", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "Column1", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "column1_int", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.Int32 }, + new TextLoader.Column(){ Name = "column1_string", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "column1", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.Int32 } } }; public static List> expectedLabels = new List> @@ -201,7 +213,7 @@ public class CodeGenTestData "public float Country{get; set;}", "\r\n", "[ColumnName(\"Country\"), LoadColumn(2)]", - "public string Country_string{get; set;}", + "public string Country_string_0{get; set;}", "\r\n" }, new List @@ -213,7 +225,7 @@ public class CodeGenTestData "public int Shape{get; set;}", "\r\n", "[ColumnName(\"Shape\"), LoadColumn(2)]", - "public string Shape_string{get; set;}", + "public string Shape_string_0{get; set;}", "\r\n", "[ColumnName(\"color\"), LoadColumn(3)]", "public string Color{get; set;}", @@ -231,22 +243,22 @@ public class CodeGenTestData "public string Make{get; set;}", "\r\n", "[ColumnName(\"make\"), LoadColumn(2)]", - "public int Make_int{get; set;}", + "public int Make_int_0{get; set;}", "\r\n", "[ColumnName(\"Model\"), LoadColumn(3)]", "public int Model{get; set;}", "\r\n", "[ColumnName(\"model\"), LoadColumn(4)]", - "public string Model_string{get; set;}", + "public string Model_string_0{get; set;}", "\r\n", "[ColumnName(\"model\"), LoadColumn(5)]", - "public int Model_int{get; set;}", + "public int Model_int_1{get; set;}", "\r\n", "[ColumnName(\"color\"), LoadColumn(6)]", "public string Color{get; set;}", "\r\n", "[ColumnName(\"Color\"), LoadColumn(7)]", - "public int Color_int{get; set;}", + "public int Color_int_0{get; set;}", "\r\n", "[ColumnName(\"MSRP\"), LoadColumn(8)]", "public float MSRP{get; set;}", @@ -264,27 +276,57 @@ public class CodeGenTestData "public string Var_text{get; set;}", "\r\n", "[ColumnName(\"var_text\"), LoadColumn(1)]", - "public string Var_text_string{get; set;}", + "public string Var_text_string_1{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(2)]", "public int Var_num{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(3)]", - "public int Var_num_int{get; set;}", + "public int Var_num_int_1{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(4)]", - "public int Var_num_int_1{get; set;}", + "public int Var_num_int_2{get; set;}", "\r\n", "[ColumnName(\"var_text\"), LoadColumn(5)]", - "public string Var_text_string_1{get; set;}", + "public string Var_text_string_2{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(6)]", - "public int Var_num_int_2{get; set;}", + "public int Var_num_int_3{get; set;}", "\r\n", "[ColumnName(\"var_text\"), LoadColumn(7)]", - "public string Var_text_string_2{get; set;}", + "public string Var_text_string_3{get; set;}", "\r\n", - } + "[ColumnName(\"var_num\"), LoadColumn(8)]", + "public double Var_num_double_0{get; set;}", + "\r\n" + }, + new List + { + "[ColumnName(\"column1\"), LoadColumn(0)]", + "public string Column1{get; set;}", + "\r\n", + "[ColumnName(\"column1_string\"), LoadColumn(1)]", + "public string Column1_string_1{get; set;}", + "\r\n", + "[ColumnName(\"column1_string_1\"), LoadColumn(2)]", + "public string Column1_string_2{get; set;}", + "\r\n", + "[ColumnName(\"column1_string_2\"), LoadColumn(3)]", + "public string Column1_string_3{get; set;}", + "\r\n", + "[ColumnName(\"Column1\"), LoadColumn(4)]", + "public string Column1_string_4{get; set;}", + "\r\n", + "[ColumnName(\"column1_int\"), LoadColumn(5)]", + "public int Column1_int_0{get; set;}", + "\r\n", + "[ColumnName(\"column1_string\"), LoadColumn(6)]", + "public string Column1_string_5{get; set;}", + "\r\n", + "[ColumnName(\"column1\"), LoadColumn(7)]", + "public int Column1_int_1{get; set;}", + "\r\n" + }, }; } } From 42dfa105fddf04ab23d0e574580edf6f99a955a6 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Mon, 22 Jun 2020 19:00:07 -0700 Subject: [PATCH 5/5] Now adding _col_x as differentiator --- src/Microsoft.ML.CodeGenerator/Utils.cs | 54 +++----- ...nputClassTest.unsanitized_map.approved.txt | 16 +-- ...lassTest.unsanitized_null_map.approved.txt | 16 +-- .../CodeGenTests.cs | 120 +++++------------- 4 files changed, 66 insertions(+), 140 deletions(-) diff --git a/src/Microsoft.ML.CodeGenerator/Utils.cs b/src/Microsoft.ML.CodeGenerator/Utils.cs index c715a99ead..6e97a4397a 100644 --- a/src/Microsoft.ML.CodeGenerator/Utils.cs +++ b/src/Microsoft.ML.CodeGenerator/Utils.cs @@ -247,8 +247,8 @@ internal static int CreateSolutionFile(string solutionFile, string outputPath) internal static IList GenerateClassLabels(ColumnInferenceResults columnInferenceResults, IDictionary columnMapping = default) { IList result = new List(); - HashSet columnNames = new HashSet(); - Dictionary<(string, DataKind), int> propertyNames = new Dictionary<(string, DataKind), int>(); + List normalizedColumnNames = new List(); + bool duplicateColumnNamesExist = false; foreach (var column in columnInferenceResults.TextLoaderOptions.Columns) { StringBuilder sb = new StringBuilder(); @@ -284,46 +284,26 @@ internal static IList GenerateClassLabels(ColumnInferenceResults columnI result.Add($"[ColumnName(\"{columnName}\"), LoadColumn({column.Source[0].Min})]"); } sb.Append(" "); - - // Obtain normalized and unique version of column name - string normalizedColumnName = GetNormalizedColumnName(column.Name, dataKind, ref propertyNames); - sb.Append(normalizedColumnName); - sb.Append("{get; set;}"); + string normalizedColumnName = Utils.Normalize(column.Name); + // Put placeholder for normalized and unique version of column name + if (!duplicateColumnNamesExist && normalizedColumnNames.Contains(normalizedColumnName)) + duplicateColumnNamesExist = true; + normalizedColumnNames.Add(normalizedColumnName); result.Add(sb.ToString()); result.Add("\r\n"); } - return result; - } - - internal static string GetNormalizedColumnName(string rawColumnName, DataKind dataKind, ref Dictionary<(string, DataKind), int> propertyNames) - { - // Get normalized column name for correctly typed class property name - string normalizedColumnName = Utils.Normalize(rawColumnName); - // Check if the normalized column name already has type and/or count in name, and remove that substring if it exists. - // These will be added on as necessary below. - // This regex checks for the "_[dataKind]" and "_[dataKind]_[int]" pattern. - Match match = Regex.Match(normalizedColumnName, $@"_{GetSymbolOfDataKind(dataKind)}_*(\d)*?$"); - if (match.Success) - normalizedColumnName = normalizedColumnName.Substring(0, match.Index); - - // Check if there's already a variable with the same normalized column name and type - if (propertyNames.ContainsKey((normalizedColumnName, dataKind))) - { - propertyNames[(normalizedColumnName, dataKind)] += 1; - normalizedColumnName += String.Concat("_", GetSymbolOfDataKind(dataKind), "_", propertyNames[(normalizedColumnName, dataKind)] - 1); - } - // Check if there's already a variable with the same normalized column name but different type - else if (propertyNames.Keys.ToList().Select(t=>t.Item1).Contains(normalizedColumnName)) - { - propertyNames.Add((normalizedColumnName, dataKind), 1); - normalizedColumnName += String.Concat("_", GetSymbolOfDataKind(dataKind), "_", propertyNames[(normalizedColumnName, dataKind)] - 1); - } - // This normalized column name is unique, add it to dictionary - else + for (int i = 1; i < result.Count; i+=3) { - propertyNames.Add((normalizedColumnName, dataKind), 1); + // Get normalized column name for correctly typed class property name + // If duplicate column names exist, the only way to ensure all generated column names are unique is to add + // a differentiator depending on the column load order from dataset. + if (duplicateColumnNamesExist) + result[i] += normalizedColumnNames[i/3] + $"_col_{i/3}"; + else + result[i] += normalizedColumnNames[i/3]; + result[i] += "{get; set;}"; } - return normalizedColumnName; + return result; } internal static string GetSymbolOfDataKind(DataKind dataKind) diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt index a8498455a3..6cdb0b2f20 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_map.approved.txt @@ -11,35 +11,35 @@ namespace test.Model public class ModelInput { [ColumnName("input_0"), LoadColumn(0)] - public int Id { get; set; } + public int Id_col_0 { get; set; } [ColumnName("input_1"), LoadColumn(1)] - public int MsAssetNum { get; set; } + public int MsAssetNum_col_1 { get; set; } [ColumnName("input_2"), LoadColumn(2)] - public string Make { get; set; } + public string Make_col_2 { get; set; } [ColumnName("input_3"), LoadColumn(3)] - public string Model { get; set; } + public string Model_col_3 { get; set; } [ColumnName("input_4"), LoadColumn(4)] - public double Model_double_0 { get; set; } + public double Model_col_4 { get; set; } [ColumnName("input_5"), LoadColumn(5)] - public string Work_category { get; set; } + public string Work_category_col_5 { get; set; } [ColumnName("Work category"), LoadColumn(6)] - public int Work_category_int_0 { get; set; } + public int Work_category_col_6 { get; set; } [ColumnName("input_7"), LoadColumn(7)] - public bool IsDetachable { get; set; } + public bool IsDetachable_col_7 { get; set; } } diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt index 08e201bba3..661d99b0cf 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ModelInputClassTest.unsanitized_null_map.approved.txt @@ -11,35 +11,35 @@ namespace test.Model public class ModelInput { [ColumnName("id"), LoadColumn(0)] - public int Id { get; set; } + public int Id_col_0 { get; set; } [ColumnName("MsAssetNum"), LoadColumn(1)] - public int MsAssetNum { get; set; } + public int MsAssetNum_col_1 { get; set; } [ColumnName("Make"), LoadColumn(2)] - public string Make { get; set; } + public string Make_col_2 { get; set; } [ColumnName("Model"), LoadColumn(3)] - public string Model { get; set; } + public string Model_col_3 { get; set; } [ColumnName("model"), LoadColumn(4)] - public double Model_double_0 { get; set; } + public double Model_col_4 { get; set; } [ColumnName("work category"), LoadColumn(5)] - public string Work_category { get; set; } + public string Work_category_col_5 { get; set; } [ColumnName("Work category"), LoadColumn(6)] - public int Work_category_int_0 { get; set; } + public int Work_category_col_6 { get; set; } [ColumnName("IsDetachable"), LoadColumn(7)] - public bool IsDetachable { get; set; } + public bool IsDetachable_col_7 { get; set; } } diff --git a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs index 4b8b465374..478d8027cb 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs +++ b/test/Microsoft.ML.CodeGenerator.Tests/CodeGenTests.cs @@ -160,17 +160,13 @@ public class CodeGenTestData }, new TextLoader.Column[] { - new TextLoader.Column(){ Name = "vin", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "Make", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "make", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "Model", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "model", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "model", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "color", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "Color", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "MSRP", Source = new TextLoader.Range[]{new TextLoader.Range(8) }, DataKind = DataKind.Single }, - new TextLoader.Column(){ Name = "engine size", Source = new TextLoader.Range[]{new TextLoader.Range(9) }, DataKind = DataKind.Double }, - new TextLoader.Column(){ Name = "isElectric", Source = new TextLoader.Range[]{new TextLoader.Range(10) }, DataKind = DataKind.Boolean }, + new TextLoader.Column(){ Name = "vin", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "make", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "model", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "color", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.String }, + new TextLoader.Column(){ Name = "MSRP", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.Single }, + new TextLoader.Column(){ Name = "engine size", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.Double }, + new TextLoader.Column(){ Name = "isElectric", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.Boolean }, }, new TextLoader.Column[] { @@ -183,17 +179,6 @@ public class CodeGenTestData new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.Int32 }, new TextLoader.Column(){ Name = "var_text", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.String }, new TextLoader.Column(){ Name = "var_num", Source = new TextLoader.Range[]{new TextLoader.Range(8) }, DataKind = DataKind.Double }, - }, - new TextLoader.Column[] - { - new TextLoader.Column(){ Name = "column1", Source = new TextLoader.Range[]{new TextLoader.Range(0) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "column1_string", Source = new TextLoader.Range[]{new TextLoader.Range(1) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "column1_string_1", Source = new TextLoader.Range[]{new TextLoader.Range(2) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "column1_string_2", Source = new TextLoader.Range[]{new TextLoader.Range(3) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "Column1", Source = new TextLoader.Range[]{new TextLoader.Range(4) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "column1_int", Source = new TextLoader.Range[]{new TextLoader.Range(5) }, DataKind = DataKind.Int32 }, - new TextLoader.Column(){ Name = "column1_string", Source = new TextLoader.Range[]{new TextLoader.Range(6) }, DataKind = DataKind.String }, - new TextLoader.Column(){ Name = "column1", Source = new TextLoader.Range[]{new TextLoader.Range(7) }, DataKind = DataKind.Int32 } } }; public static List> expectedLabels = new List> @@ -207,126 +192,87 @@ public class CodeGenTestData new List { "[ColumnName(\"id\"), LoadColumn(0)]", - "public float Id{get; set;}", + "public float Id_col_0{get; set;}", "\r\n", "[ColumnName(\"country\"), LoadColumn(1)]", - "public float Country{get; set;}", + "public float Country_col_1{get; set;}", "\r\n", "[ColumnName(\"Country\"), LoadColumn(2)]", - "public string Country_string_0{get; set;}", + "public string Country_col_2{get; set;}", "\r\n" }, new List { "[ColumnName(\"id\"), LoadColumn(0)]", - "public int Id{get; set;}", + "public int Id_col_0{get; set;}", "\r\n", "[ColumnName(\"shape\"), LoadColumn(1)]", - "public int Shape{get; set;}", + "public int Shape_col_1{get; set;}", "\r\n", "[ColumnName(\"Shape\"), LoadColumn(2)]", - "public string Shape_string_0{get; set;}", + "public string Shape_col_2{get; set;}", "\r\n", "[ColumnName(\"color\"), LoadColumn(3)]", - "public string Color{get; set;}", + "public string Color_col_3{get; set;}", "\r\n", "[ColumnName(\"price\"), LoadColumn(4)]", - "public double Price{get; set;}", + "public double Price_col_4{get; set;}", "\r\n" }, new List { "[ColumnName(\"vin\"), LoadColumn(0)]", - "public int Vin{get; set;}", + "public string Vin{get; set;}", "\r\n", - "[ColumnName(\"Make\"), LoadColumn(1)]", + "[ColumnName(\"make\"), LoadColumn(1)]", "public string Make{get; set;}", "\r\n", - "[ColumnName(\"make\"), LoadColumn(2)]", - "public int Make_int_0{get; set;}", - "\r\n", - "[ColumnName(\"Model\"), LoadColumn(3)]", - "public int Model{get; set;}", + "[ColumnName(\"model\"), LoadColumn(2)]", + "public string Model{get; set;}", "\r\n", - "[ColumnName(\"model\"), LoadColumn(4)]", - "public string Model_string_0{get; set;}", - "\r\n", - "[ColumnName(\"model\"), LoadColumn(5)]", - "public int Model_int_1{get; set;}", - "\r\n", - "[ColumnName(\"color\"), LoadColumn(6)]", + "[ColumnName(\"color\"), LoadColumn(3)]", "public string Color{get; set;}", "\r\n", - "[ColumnName(\"Color\"), LoadColumn(7)]", - "public int Color_int_0{get; set;}", - "\r\n", - "[ColumnName(\"MSRP\"), LoadColumn(8)]", + "[ColumnName(\"MSRP\"), LoadColumn(4)]", "public float MSRP{get; set;}", "\r\n", - "[ColumnName(\"engine size\"), LoadColumn(9)]", + "[ColumnName(\"engine size\"), LoadColumn(5)]", "public double Engine_size{get; set;}", "\r\n", - "[ColumnName(\"isElectric\"), LoadColumn(10)]", + "[ColumnName(\"isElectric\"), LoadColumn(6)]", "public bool IsElectric{get; set;}", "\r\n" }, new List { "[ColumnName(\"var_text\"), LoadColumn(0)]", - "public string Var_text{get; set;}", + "public string Var_text_col_0{get; set;}", "\r\n", "[ColumnName(\"var_text\"), LoadColumn(1)]", - "public string Var_text_string_1{get; set;}", + "public string Var_text_col_1{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(2)]", - "public int Var_num{get; set;}", + "public int Var_num_col_2{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(3)]", - "public int Var_num_int_1{get; set;}", + "public int Var_num_col_3{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(4)]", - "public int Var_num_int_2{get; set;}", + "public int Var_num_col_4{get; set;}", "\r\n", "[ColumnName(\"var_text\"), LoadColumn(5)]", - "public string Var_text_string_2{get; set;}", + "public string Var_text_col_5{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(6)]", - "public int Var_num_int_3{get; set;}", + "public int Var_num_col_6{get; set;}", "\r\n", "[ColumnName(\"var_text\"), LoadColumn(7)]", - "public string Var_text_string_3{get; set;}", + "public string Var_text_col_7{get; set;}", "\r\n", "[ColumnName(\"var_num\"), LoadColumn(8)]", - "public double Var_num_double_0{get; set;}", + "public double Var_num_col_8{get; set;}", "\r\n" - }, - new List - { - "[ColumnName(\"column1\"), LoadColumn(0)]", - "public string Column1{get; set;}", - "\r\n", - "[ColumnName(\"column1_string\"), LoadColumn(1)]", - "public string Column1_string_1{get; set;}", - "\r\n", - "[ColumnName(\"column1_string_1\"), LoadColumn(2)]", - "public string Column1_string_2{get; set;}", - "\r\n", - "[ColumnName(\"column1_string_2\"), LoadColumn(3)]", - "public string Column1_string_3{get; set;}", - "\r\n", - "[ColumnName(\"Column1\"), LoadColumn(4)]", - "public string Column1_string_4{get; set;}", - "\r\n", - "[ColumnName(\"column1_int\"), LoadColumn(5)]", - "public int Column1_int_0{get; set;}", - "\r\n", - "[ColumnName(\"column1_string\"), LoadColumn(6)]", - "public string Column1_string_5{get; set;}", - "\r\n", - "[ColumnName(\"column1\"), LoadColumn(7)]", - "public int Column1_int_1{get; set;}", - "\r\n" - }, + } }; } }