From 2519662d10afd081f8729b909c6002ce6e1b4fd9 Mon Sep 17 00:00:00 2001 From: Lynx1820 Date: Wed, 17 Jun 2020 15:00:40 -0700 Subject: [PATCH 1/3] custom groupid --- src/Microsoft.ML.AutoML/API/ExperimentBase.cs | 6 +++--- .../Experiment/MetricsAgents/BinaryMetricsAgent.cs | 2 +- .../Experiment/MetricsAgents/IMetricsAgent.cs | 2 +- .../Experiment/MetricsAgents/MultiMetricsAgent.cs | 2 +- .../Experiment/MetricsAgents/RankingMetricsAgent.cs | 4 ++-- .../Experiment/MetricsAgents/RegressionMetricsAgent.cs | 2 +- .../Experiment/Runners/CrossValRunner.cs | 5 ++++- .../Experiment/Runners/CrossValSummaryRunner.cs | 5 ++++- src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs | 3 ++- .../Experiment/Runners/TrainValidateRunner.cs | 5 ++++- .../TrainerExtensions/TrainerExtensionUtil.cs | 2 +- test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- 12 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentBase.cs b/src/Microsoft.ML.AutoML/API/ExperimentBase.cs index b1d33781ae..69007f0aaa 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentBase.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentBase.cs @@ -253,7 +253,7 @@ private ExperimentResult ExecuteTrainValidate(IDataView trainData, validationData = preprocessorTransform.Transform(validationData); } - var runner = new TrainValidateRunner(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent, + var runner = new TrainValidateRunner(Context, trainData, validationData, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, MetricsAgent, preFeaturizer, preprocessorTransform, _logger); var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo); return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner); @@ -273,7 +273,7 @@ private CrossValidationExperimentResult ExecuteCrossVal(IDataView[] tr (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer); var runner = new CrossValRunner(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer, - preprocessorTransforms, columnInfo.LabelColumnName, _logger); + preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, _logger); var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo); // Execute experiment & get all pipelines run @@ -300,7 +300,7 @@ private ExperimentResult ExecuteCrossValSummary(IDataView[] trainDatas (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer); var runner = new CrossValSummaryRunner(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer, - preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger); + preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, OptimizingMetricInfo, _logger); var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo); return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner); } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/BinaryMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/BinaryMetricsAgent.cs index e6e0451e9b..bf2a855409 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/BinaryMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/BinaryMetricsAgent.cs @@ -78,7 +78,7 @@ public bool IsModelPerfect(double score) } } - public BinaryClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn) + public BinaryClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId) { return _mlContext.BinaryClassification.EvaluateNonCalibrated(data, labelColumn); } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/IMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/IMetricsAgent.cs index 3531809ba4..e2af9797c8 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/IMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/IMetricsAgent.cs @@ -10,6 +10,6 @@ internal interface IMetricsAgent bool IsModelPerfect(double score); - T EvaluateMetrics(IDataView data, string labelColumn); + T EvaluateMetrics(IDataView data, string labelColumn, string groupId); } } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs index abd584ae53..e37d2ce63d 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs @@ -66,7 +66,7 @@ public bool IsModelPerfect(double score) } } - public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn) + public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId) { return _mlContext.MulticlassClassification.Evaluate(data, labelColumn); } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs index 0a0c574ce3..0544a6b638 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs @@ -57,9 +57,9 @@ public bool IsModelPerfect(double score) } } - public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn) + public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupIdColumn) { - return _mlContext.Ranking.Evaluate(data, labelColumn); + return _mlContext.Ranking.Evaluate(data, labelColumn, groupIdColumn); } } } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs index 94f5fcdae7..cedf57c145 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs @@ -61,7 +61,7 @@ public bool IsModelPerfect(double score) } } - public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn) + public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId) { return _mlContext.Regression.Evaluate(data, labelColumn); } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs index 98e54d4b46..83be249561 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs @@ -20,6 +20,7 @@ internal class CrossValRunner : IRunner _preFeaturizer; private readonly ITransformer[] _preprocessorTransforms; private readonly string _labelColumn; + private readonly string _groupIdColumn; private readonly IChannel _logger; private readonly DataViewSchema _modelInputSchema; @@ -30,6 +31,7 @@ public CrossValRunner(MLContext context, IEstimator preFeaturizer, ITransformer[] preprocessorTransforms, string labelColumn, + string groupIdColumn, IChannel logger) { _context = context; @@ -39,6 +41,7 @@ public CrossValRunner(MLContext context, _preFeaturizer = preFeaturizer; _preprocessorTransforms = preprocessorTransforms; _labelColumn = labelColumn; + _groupIdColumn = groupIdColumn; _logger = logger; _modelInputSchema = trainDatasets[0].Schema; } @@ -52,7 +55,7 @@ public CrossValRunner(MLContext context, { var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1); var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i], - _labelColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger); + _labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger); trainResults.Add(new SuggestedPipelineTrainResult(trainResult.model, trainResult.metrics, trainResult.exception, trainResult.score)); } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index eb5c02f89a..13a533cc82 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -21,6 +21,7 @@ internal class CrossValSummaryRunner : IRunner> private readonly IEstimator _preFeaturizer; private readonly ITransformer[] _preprocessorTransforms; private readonly string _labelColumn; + private readonly string _groupIdColumn; private readonly OptimizingMetricInfo _optimizingMetricInfo; private readonly IChannel _logger; private readonly DataViewSchema _modelInputSchema; @@ -32,6 +33,7 @@ public CrossValSummaryRunner(MLContext context, IEstimator preFeaturizer, ITransformer[] preprocessorTransforms, string labelColumn, + string groupIdColumn, OptimizingMetricInfo optimizingMetricInfo, IChannel logger) { @@ -42,6 +44,7 @@ public CrossValSummaryRunner(MLContext context, _preFeaturizer = preFeaturizer; _preprocessorTransforms = preprocessorTransforms; _labelColumn = labelColumn; + _groupIdColumn = groupIdColumn; _optimizingMetricInfo = optimizingMetricInfo; _logger = logger; _modelInputSchema = trainDatasets[0].Schema; @@ -56,7 +59,7 @@ public CrossValSummaryRunner(MLContext context, { var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1); var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i], - _labelColumn, _metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema, + _labelColumn, _groupIdColumn,_metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema, _logger); trainResults.Add(trainResult); } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index c6aeef49a5..7a09ae1e25 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -16,6 +16,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub IDataView trainData, IDataView validData, string labelColumn, + string groupId, IMetricsAgent metricsAgent, ITransformer preprocessorTransform, FileInfo modelFileInfo, @@ -28,7 +29,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub var model = estimator.Fit(trainData); var scoredData = model.Transform(validData); - var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn); + var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn, groupId); var score = metricsAgent.GetScore(metrics); if (preprocessorTransform != null) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs index d608f7dd2f..a1cc6ab48b 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs @@ -14,6 +14,7 @@ internal class TrainValidateRunner : IRunner> private readonly IDataView _trainData; private readonly IDataView _validData; private readonly string _labelColumn; + private readonly string _groupIdColumn; private readonly IMetricsAgent _metricsAgent; private readonly IEstimator _preFeaturizer; private readonly ITransformer _preprocessorTransform; @@ -24,6 +25,7 @@ public TrainValidateRunner(MLContext context, IDataView trainData, IDataView validData, string labelColumn, + string groupIdColumn, IMetricsAgent metricsAgent, IEstimator preFeaturizer, ITransformer preprocessorTransform, @@ -33,6 +35,7 @@ public TrainValidateRunner(MLContext context, _trainData = trainData; _validData = validData; _labelColumn = labelColumn; + _groupIdColumn = groupIdColumn; _metricsAgent = metricsAgent; _preFeaturizer = preFeaturizer; _preprocessorTransform = preprocessorTransform; @@ -45,7 +48,7 @@ public TrainValidateRunner(MLContext context, { var modelFileInfo = GetModelFileInfo(modelDirectory, iterationNum); var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainData, _validData, - _labelColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger); + _labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger); var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail(pipeline, trainResult.score, trainResult.exception == null, diff --git a/src/Microsoft.ML.AutoML/TrainerExtensions/TrainerExtensionUtil.cs b/src/Microsoft.ML.AutoML/TrainerExtensions/TrainerExtensionUtil.cs index eab10e5def..7d8d55e49a 100644 --- a/src/Microsoft.ML.AutoML/TrainerExtensions/TrainerExtensionUtil.cs +++ b/src/Microsoft.ML.AutoML/TrainerExtensions/TrainerExtensionUtil.cs @@ -158,7 +158,7 @@ private static IDictionary BuildBasePipelineNodeProps(IEnumerabl } private static IDictionary BuildLightGbmPipelineNodeProps(IEnumerable sweepParams, - string labelColumn, string weightColumn, string groupColumn = null) + string labelColumn, string weightColumn, string groupColumn) { Dictionary props = null; if (sweepParams == null || !sweepParams.Any()) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index b43acf7af1..4a6bfa11e4 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -124,7 +124,7 @@ public void AutoFitRankingTest() { string labelColumnName = "Label"; string scoreColumnName = "Score"; - string groupIdColumnName = "GroupId"; + string groupIdColumnName = "CustomGroupId"; string featuresColumnVectorNameA = "FeatureVectorA"; string featuresColumnVectorNameB = "FeatureVectorB"; var mlContext = new MLContext(1); From 63e4d51a9e8fa24a97d81c11c0ec4184679a84ba Mon Sep 17 00:00:00 2001 From: Lynx1820 Date: Mon, 29 Jun 2020 16:59:21 -0700 Subject: [PATCH 2/3] groupid addition option 2 --- .../API/RankingExperiment.cs | 19 +++++++++++---- .../MetricsAgents/BinaryMetricsAgent.cs | 2 +- .../Experiment/MetricsAgents/IMetricsAgent.cs | 2 +- .../MetricsAgents/MultiMetricsAgent.cs | 2 +- .../MetricsAgents/RankingMetricsAgent.cs | 8 ++++--- .../MetricsAgents/RegressionMetricsAgent.cs | 2 +- .../Experiment/Runners/CrossValRunner.cs | 4 ++-- .../Runners/CrossValSummaryRunner.cs | 2 +- .../Experiment/Runners/RunnerUtil.cs | 3 +-- .../Experiment/Runners/TrainValidateRunner.cs | 2 +- .../Utils/BestResultUtil.cs | 4 ++-- .../Microsoft.ML.AutoML.Tests/AutoFitTests.cs | 2 +- .../MetricsAgentsTests.cs | 24 +++++++++---------- 13 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/RankingExperiment.cs b/src/Microsoft.ML.AutoML/API/RankingExperiment.cs index ea0ba735c4..7233dae9b4 100644 --- a/src/Microsoft.ML.AutoML/API/RankingExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/RankingExperiment.cs @@ -19,6 +19,12 @@ public sealed class RankingExperimentSettings : ExperimentSettings /// The default value is . public RankingMetric OptimizingMetric { get; set; } + /// + /// Name for the GroupId column. + /// + /// The default value is GroupId. + public string GroupIdColumnName { get; set; } + /// /// Collection of trainers the AutoML experiment can leverage. /// @@ -28,6 +34,7 @@ public sealed class RankingExperimentSettings : ExperimentSettings public ICollection Trainers { get; } public RankingExperimentSettings() { + GroupIdColumnName = "GroupId"; OptimizingMetric = RankingMetric.Ndcg; Trainers = Enum.GetValues(typeof(RankingTrainer)).OfType().ToList(); } @@ -68,10 +75,11 @@ public static class RankingExperimentResultExtensions /// /// Enumeration of AutoML experiment run results. /// Metric to consider when selecting the best run. + /// Name for the GroupId column. /// The best experiment run. - public static RunDetail Best(this IEnumerable> results, RankingMetric metric = RankingMetric.Ndcg) + public static RunDetail Best(this IEnumerable> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId") { - var metricsAgent = new RankingMetricsAgent(null, metric); + var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName); var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing; return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing); } @@ -81,10 +89,11 @@ public static RunDetail Best(this IEnumerable /// Enumeration of AutoML experiment cross validation run results. /// Metric to consider when selecting the best run. + /// Name for the GroupId column. /// The best experiment run. - public static CrossValidationRunDetail Best(this IEnumerable> results, RankingMetric metric = RankingMetric.Ndcg) + public static CrossValidationRunDetail Best(this IEnumerable> results, RankingMetric metric = RankingMetric.Ndcg, string groupIdColumnName = "GroupId") { - var metricsAgent = new RankingMetricsAgent(null, metric); + var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName); var isMetricMaximizing = new OptimizingMetricInfo(metric).IsMaximizing; return BestResultUtil.GetBestRun(results, metricsAgent, isMetricMaximizing); } @@ -103,7 +112,7 @@ public sealed class RankingExperiment : ExperimentBase bool IsModelPerfect(double score); - T EvaluateMetrics(IDataView data, string labelColumn, string groupId); + T EvaluateMetrics(IDataView data, string labelColumn); } } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs index e37d2ce63d..abd584ae53 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/MultiMetricsAgent.cs @@ -66,7 +66,7 @@ public bool IsModelPerfect(double score) } } - public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId) + public MulticlassClassificationMetrics EvaluateMetrics(IDataView data, string labelColumn) { return _mlContext.MulticlassClassification.Evaluate(data, labelColumn); } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs index 0544a6b638..f6cc162d98 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RankingMetricsAgent.cs @@ -10,11 +10,13 @@ internal class RankingMetricsAgent : IMetricsAgent { private readonly MLContext _mlContext; private readonly RankingMetric _optimizingMetric; + private readonly string _groupIdColumnName; - public RankingMetricsAgent(MLContext mlContext, RankingMetric optimizingMetric) + public RankingMetricsAgent(MLContext mlContext, RankingMetric optimizingMetric, string groupIdColumnName) { _mlContext = mlContext; _optimizingMetric = optimizingMetric; + _groupIdColumnName = groupIdColumnName; } // Optimizing metric used: NDCG@10 and DCG@10 @@ -57,9 +59,9 @@ public bool IsModelPerfect(double score) } } - public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupIdColumn) + public RankingMetrics EvaluateMetrics(IDataView data, string labelColumn) { - return _mlContext.Ranking.Evaluate(data, labelColumn, groupIdColumn); + return _mlContext.Ranking.Evaluate(data, labelColumn, _groupIdColumnName); } } } diff --git a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs index cedf57c145..94f5fcdae7 100644 --- a/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs +++ b/src/Microsoft.ML.AutoML/Experiment/MetricsAgents/RegressionMetricsAgent.cs @@ -61,7 +61,7 @@ public bool IsModelPerfect(double score) } } - public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn, string groupId) + public RegressionMetrics EvaluateMetrics(IDataView data, string labelColumn) { return _mlContext.Regression.Evaluate(data, labelColumn); } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs index 83be249561..f02e5af0fc 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs @@ -30,8 +30,8 @@ public CrossValRunner(MLContext context, IMetricsAgent metricsAgent, IEstimator preFeaturizer, ITransformer[] preprocessorTransforms, - string labelColumn, string groupIdColumn, + string labelColumn, IChannel logger) { _context = context; @@ -55,7 +55,7 @@ public CrossValRunner(MLContext context, { var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1); var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i], - _labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger); + _labelColumn, _metricsAgent, _preprocessorTransforms?[i], modelFileInfo, _modelInputSchema, _logger); trainResults.Add(new SuggestedPipelineTrainResult(trainResult.model, trainResult.metrics, trainResult.exception, trainResult.score)); } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 13a533cc82..806a5ed07d 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -59,7 +59,7 @@ public CrossValSummaryRunner(MLContext context, { var modelFileInfo = RunnerUtil.GetModelFileInfo(modelDirectory, iterationNum, i + 1); var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainDatasets[i], _validDatasets[i], - _labelColumn, _groupIdColumn,_metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema, + _labelColumn, _metricsAgent, _preprocessorTransforms?.ElementAt(i), modelFileInfo, _modelInputSchema, _logger); trainResults.Add(trainResult); } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs index 7a09ae1e25..c6aeef49a5 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/RunnerUtil.cs @@ -16,7 +16,6 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub IDataView trainData, IDataView validData, string labelColumn, - string groupId, IMetricsAgent metricsAgent, ITransformer preprocessorTransform, FileInfo modelFileInfo, @@ -29,7 +28,7 @@ public static (ModelContainer model, TMetrics metrics, Exception exception, doub var model = estimator.Fit(trainData); var scoredData = model.Transform(validData); - var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn, groupId); + var metrics = metricsAgent.EvaluateMetrics(scoredData, labelColumn); var score = metricsAgent.GetScore(metrics); if (preprocessorTransform != null) diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs index a1cc6ab48b..b539391716 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs @@ -48,7 +48,7 @@ public TrainValidateRunner(MLContext context, { var modelFileInfo = GetModelFileInfo(modelDirectory, iterationNum); var trainResult = RunnerUtil.TrainAndScorePipeline(_context, pipeline, _trainData, _validData, - _labelColumn, _groupIdColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger); + _labelColumn, _metricsAgent, _preprocessorTransform, modelFileInfo, _modelInputSchema, _logger); var suggestedPipelineRunDetail = new SuggestedPipelineRunDetail(pipeline, trainResult.score, trainResult.exception == null, diff --git a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs index a0bff55c9e..601fa212be 100644 --- a/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs +++ b/src/Microsoft.ML.AutoML/Utils/BestResultUtil.cs @@ -35,9 +35,9 @@ public static RunDetail GetBestRun(IEnumerable< } public static RunDetail GetBestRun(IEnumerable> results, - RankingMetric metric) + RankingMetric metric, string groupIdColumnName) { - var metricsAgent = new RankingMetricsAgent(null, metric); + var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName); var metricInfo = new OptimizingMetricInfo(metric); return GetBestRun(results, metricsAgent, metricInfo.IsMaximizing); } diff --git a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs index 4a6bfa11e4..97b3e07aa4 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs @@ -136,7 +136,7 @@ public void AutoFitRankingTest() trainDataView = mlContext.Data.SkipRows(trainDataView, 500); // STEP 2: Run AutoML experiment ExperimentResult experimentResult = mlContext.Auto() - .CreateRankingExperiment(5) + .CreateRankingExperiment(new RankingExperimentSettings() { GroupIdColumnName = "CustomGroupId", MaxExperimentTimeInSeconds = 5}) .Execute(trainDataView, testDataView, new ColumnInformation() { diff --git a/test/Microsoft.ML.AutoML.Tests/MetricsAgentsTests.cs b/test/Microsoft.ML.AutoML.Tests/MetricsAgentsTests.cs index 0f9b336d84..2f3745d937 100644 --- a/test/Microsoft.ML.AutoML.Tests/MetricsAgentsTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/MetricsAgentsTests.cs @@ -127,14 +127,14 @@ public void RankingMetricsGetScoreTest() double[] ndcg = { 0.2, 0.3, 0.4 }; double[] dcg = { 0.2, 0.3, 0.4 }; var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg); - Assert.Equal(0.4, GetScore(metrics, RankingMetric.Dcg)); - Assert.Equal(0.4, GetScore(metrics, RankingMetric.Ndcg)); + Assert.Equal(0.4, GetScore(metrics, RankingMetric.Dcg, "GroupId")); + Assert.Equal(0.4, GetScore(metrics, RankingMetric.Ndcg, "GroupId")); double[] largeNdcg = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95 }; double[] largeDcg = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.85, 0.9, 0.95 }; metrics = MetricsUtil.CreateRankingMetrics(largeDcg, largeNdcg); - Assert.Equal(0.9, GetScore(metrics, RankingMetric.Dcg)); - Assert.Equal(0.9, GetScore(metrics, RankingMetric.Ndcg)); + Assert.Equal(0.9, GetScore(metrics, RankingMetric.Dcg, "GroupId")); + Assert.Equal(0.9, GetScore(metrics, RankingMetric.Ndcg, "GroupId")); } [Fact] @@ -143,8 +143,8 @@ public void RankingMetricsNonPerfectTest() double[] ndcg = { 0.2, 0.3, 0.4 }; double[] dcg = { 0.2, 0.3, 0.4 }; var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg); - Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg)); - Assert.False(IsPerfectModel(metrics, RankingMetric.Ndcg)); + Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg, "GroupId")); + Assert.False(IsPerfectModel(metrics, RankingMetric.Ndcg, "GroupId")); } [Fact] @@ -153,8 +153,8 @@ public void RankingMetricsPerfectTest() double[] ndcg = { 0.2, 0.3, 1 }; double[] dcg = { 0.2, 0.3, 1 }; var metrics = MetricsUtil.CreateRankingMetrics(dcg, ndcg); - Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg)); //REVIEW: No true Perfect model - Assert.True(IsPerfectModel(metrics, RankingMetric.Ndcg)); + Assert.False(IsPerfectModel(metrics, RankingMetric.Dcg, "GroupId")); //REVIEW: No true Perfect model + Assert.True(IsPerfectModel(metrics, RankingMetric.Ndcg, "GroupId")); } [Fact] @@ -179,9 +179,9 @@ private static double GetScore(RegressionMetrics metrics, RegressionMetric metri return new RegressionMetricsAgent(null, metric).GetScore(metrics); } - private static double GetScore(RankingMetrics metrics, RankingMetric metric) + private static double GetScore(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName) { - return new RankingMetricsAgent(null, metric).GetScore(metrics); + return new RankingMetricsAgent(null, metric, groupIdColumnName).GetScore(metrics); } private static bool IsPerfectModel(BinaryClassificationMetrics metrics, BinaryClassificationMetric metric) @@ -202,9 +202,9 @@ private static bool IsPerfectModel(RegressionMetrics metrics, RegressionMetric m return IsPerfectModel(metricsAgent, metrics); } - private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric) + private static bool IsPerfectModel(RankingMetrics metrics, RankingMetric metric, string groupIdColumnName) { - var metricsAgent = new RankingMetricsAgent(null, metric); + var metricsAgent = new RankingMetricsAgent(null, metric, groupIdColumnName); return IsPerfectModel(metricsAgent, metrics); } From a19bbca2e614b68b3831953206ea6530b8072bfd Mon Sep 17 00:00:00 2001 From: Lynx1820 Date: Mon, 29 Jun 2020 17:48:37 -0700 Subject: [PATCH 3/3] removed prev code --- src/Microsoft.ML.AutoML/API/ExperimentBase.cs | 6 +++--- .../Experiment/Runners/CrossValRunner.cs | 3 --- .../Experiment/Runners/CrossValSummaryRunner.cs | 3 --- .../Experiment/Runners/TrainValidateRunner.cs | 3 --- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/ExperimentBase.cs b/src/Microsoft.ML.AutoML/API/ExperimentBase.cs index 69007f0aaa..b1d33781ae 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentBase.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentBase.cs @@ -253,7 +253,7 @@ private ExperimentResult ExecuteTrainValidate(IDataView trainData, validationData = preprocessorTransform.Transform(validationData); } - var runner = new TrainValidateRunner(Context, trainData, validationData, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, MetricsAgent, + var runner = new TrainValidateRunner(Context, trainData, validationData, columnInfo.LabelColumnName, MetricsAgent, preFeaturizer, preprocessorTransform, _logger); var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainData, columnInfo); return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner); @@ -273,7 +273,7 @@ private CrossValidationExperimentResult ExecuteCrossVal(IDataView[] tr (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer); var runner = new CrossValRunner(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer, - preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, _logger); + preprocessorTransforms, columnInfo.LabelColumnName, _logger); var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo); // Execute experiment & get all pipelines run @@ -300,7 +300,7 @@ private ExperimentResult ExecuteCrossValSummary(IDataView[] trainDatas (trainDatasets, validationDatasets, preprocessorTransforms) = ApplyPreFeaturizerCrossVal(trainDatasets, validationDatasets, preFeaturizer); var runner = new CrossValSummaryRunner(Context, trainDatasets, validationDatasets, MetricsAgent, preFeaturizer, - preprocessorTransforms, columnInfo.LabelColumnName, columnInfo.GroupIdColumnName, OptimizingMetricInfo, _logger); + preprocessorTransforms, columnInfo.LabelColumnName, OptimizingMetricInfo, _logger); var columns = DatasetColumnInfoUtil.GetDatasetColumnInfo(Context, trainDatasets[0], columnInfo); return Execute(columnInfo, columns, preFeaturizer, progressHandler, runner); } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs index f02e5af0fc..98e54d4b46 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValRunner.cs @@ -20,7 +20,6 @@ internal class CrossValRunner : IRunner _preFeaturizer; private readonly ITransformer[] _preprocessorTransforms; private readonly string _labelColumn; - private readonly string _groupIdColumn; private readonly IChannel _logger; private readonly DataViewSchema _modelInputSchema; @@ -30,7 +29,6 @@ public CrossValRunner(MLContext context, IMetricsAgent metricsAgent, IEstimator preFeaturizer, ITransformer[] preprocessorTransforms, - string groupIdColumn, string labelColumn, IChannel logger) { @@ -41,7 +39,6 @@ public CrossValRunner(MLContext context, _preFeaturizer = preFeaturizer; _preprocessorTransforms = preprocessorTransforms; _labelColumn = labelColumn; - _groupIdColumn = groupIdColumn; _logger = logger; _modelInputSchema = trainDatasets[0].Schema; } diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs index 806a5ed07d..eb5c02f89a 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/CrossValSummaryRunner.cs @@ -21,7 +21,6 @@ internal class CrossValSummaryRunner : IRunner> private readonly IEstimator _preFeaturizer; private readonly ITransformer[] _preprocessorTransforms; private readonly string _labelColumn; - private readonly string _groupIdColumn; private readonly OptimizingMetricInfo _optimizingMetricInfo; private readonly IChannel _logger; private readonly DataViewSchema _modelInputSchema; @@ -33,7 +32,6 @@ public CrossValSummaryRunner(MLContext context, IEstimator preFeaturizer, ITransformer[] preprocessorTransforms, string labelColumn, - string groupIdColumn, OptimizingMetricInfo optimizingMetricInfo, IChannel logger) { @@ -44,7 +42,6 @@ public CrossValSummaryRunner(MLContext context, _preFeaturizer = preFeaturizer; _preprocessorTransforms = preprocessorTransforms; _labelColumn = labelColumn; - _groupIdColumn = groupIdColumn; _optimizingMetricInfo = optimizingMetricInfo; _logger = logger; _modelInputSchema = trainDatasets[0].Schema; diff --git a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs index b539391716..d608f7dd2f 100644 --- a/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs +++ b/src/Microsoft.ML.AutoML/Experiment/Runners/TrainValidateRunner.cs @@ -14,7 +14,6 @@ internal class TrainValidateRunner : IRunner> private readonly IDataView _trainData; private readonly IDataView _validData; private readonly string _labelColumn; - private readonly string _groupIdColumn; private readonly IMetricsAgent _metricsAgent; private readonly IEstimator _preFeaturizer; private readonly ITransformer _preprocessorTransform; @@ -25,7 +24,6 @@ public TrainValidateRunner(MLContext context, IDataView trainData, IDataView validData, string labelColumn, - string groupIdColumn, IMetricsAgent metricsAgent, IEstimator preFeaturizer, ITransformer preprocessorTransform, @@ -35,7 +33,6 @@ public TrainValidateRunner(MLContext context, _trainData = trainData; _validData = validData; _labelColumn = labelColumn; - _groupIdColumn = groupIdColumn; _metricsAgent = metricsAgent; _preFeaturizer = preFeaturizer; _preprocessorTransform = preprocessorTransform;