From 49e7af622332e87b6fe704d4dbce4c1c9dcebfaa Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 01:36:03 -0700 Subject: [PATCH 1/8] Updated libmf and corresponding MatrixFactorizationSimpleTrainAndPredict() baselines per build --- src/Native/MatrixFactorizationNative/libmf | 2 +- .../MatrixFactorizationTests.cs | 28 ++++++++----------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/Native/MatrixFactorizationNative/libmf b/src/Native/MatrixFactorizationNative/libmf index 298715a4e4..403153ca20 160000 --- a/src/Native/MatrixFactorizationNative/libmf +++ b/src/Native/MatrixFactorizationNative/libmf @@ -1 +1 @@ -Subproject commit 298715a4e458bc09c6a27c8643a58095afbdadf1 +Subproject commit 403153ca204817e2901b2872d977088316360641 diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 09ce334a79..5fa2f12b65 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -54,7 +54,6 @@ public void MatrixFactorization_Estimator() } [MatrixFactorizationFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined public void MatrixFactorizationSimpleTrainAndPredict() { var mlContext = new MLContext(seed: 1); @@ -96,10 +95,10 @@ public void MatrixFactorizationSimpleTrainAndPredict() // MF produce different matrices on different platforms, so check their content on Windows. if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - Assert.Equal(0.290507137775421, leftMatrix[0], 5); - Assert.Equal(0.558072924613953, leftMatrix[leftMatrix.Count - 1], 5); - Assert.Equal(0.270811557769775, rightMatrix[0], 5); - Assert.Equal(0.376706808805466, rightMatrix[rightMatrix.Count - 1], 5); + Assert.Equal(0.309137582778931, leftMatrix[0], 5); + Assert.Equal(0.468956589698792, leftMatrix[leftMatrix.Count - 1], 5); + Assert.Equal(0.303486406803131, rightMatrix[0], 5); + Assert.Equal(0.503888845443726, rightMatrix[rightMatrix.Count - 1], 5); } // Read the test data set as an IDataView var testData = reader.Load(new MultiFileSource(GetDataPath(TestDatasets.trivialMatrixFactorization.testFilename))); @@ -122,28 +121,25 @@ public void MatrixFactorizationSimpleTrainAndPredict() // Compute prediction errors var metrices = mlContext.Recommendation().Evaluate(prediction, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); - // Determine if the selected metric is reasonable for different platforms - // Windows tolerance is set at 1e-7, and Linux tolerance is set at 1e-5 - double windowsTolerance = Math.Pow(10, -7); - double linuxTolerance = Math.Pow(10, -5); + // Determine if the selected mean-squared error metric is reasonable on different platforms + double tolerance = Math.Pow(10, -7); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case - var expectedUnixL2Error = 0.610332110253861; // Linux baseline - Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - linuxTolerance, expectedUnixL2Error + linuxTolerance); + var expectedUnixL2Error = 0.612726002827395; // Linux baseline + Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - tolerance, expectedUnixL2Error + tolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - // The Mac case is just broken. Should be fixed later. Re-enable when done. // Mac case - //var expectedMacL2Error = 0.61192207960271; // Mac baseline - //Assert.InRange(metrices.L2, expectedMacL2Error - 5e-3, expectedMacL2Error + 5e-3); // 1e-7 is too small for Mac so we try 1e-5 + var expectedMacL2Error = 0.616389336408704; // Mac baseline + Assert.InRange(metrices.MeanSquaredError, expectedMacL2Error - tolerance, expectedMacL2Error + tolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // Windows case - var expectedWindowsL2Error = 0.60226203382884; // Windows baseline - Assert.InRange(metrices.MeanSquaredError, expectedWindowsL2Error - windowsTolerance, expectedWindowsL2Error + windowsTolerance); + var expectedWindowsL2Error = 0.600329985097577; // Windows baseline + Assert.InRange(metrices.MeanSquaredError, expectedWindowsL2Error - tolerance, expectedWindowsL2Error + tolerance); } var modelWithValidation = pipeline.Fit(data, testData); From ca3f3d0a3111456858aa14d8430ea2c1fd6f7f3b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 02:44:46 -0700 Subject: [PATCH 2/8] Updated for slight variance in CentOS 7 --- .../MatrixFactorizationTests.cs | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 5fa2f12b65..a88992f0ea 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -121,25 +122,34 @@ public void MatrixFactorizationSimpleTrainAndPredict() // Compute prediction errors var metrices = mlContext.Recommendation().Evaluate(prediction, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); - // Determine if the selected mean-squared error metric is reasonable on different platforms - double tolerance = Math.Pow(10, -7); + // Determine if the selected mean-squared error metric is reasonable on different platforms within the variation tolerance. + int variationTolerance = 7; if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case - var expectedUnixL2Error = 0.612726002827395; // Linux baseline - Assert.InRange(metrices.MeanSquaredError, expectedUnixL2Error - tolerance, expectedUnixL2Error + tolerance); + if (OsIsCentOS7()) + { + double expectedCentOS7LinuxMeanSquaredError = 0.612732360518435; // CentOS 7 Linux baseline + Assert.Equal(metrices.MeanSquaredError, expectedCentOS7LinuxMeanSquaredError, variationTolerance); + } + else + { + double expectedUbuntuLinuxMeanSquaredError = 0.612726002827395; // Ubuntu Linux baseline + Assert.Equal(metrices.MeanSquaredError, expectedUbuntuLinuxMeanSquaredError, variationTolerance); + } + } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { // Mac case - var expectedMacL2Error = 0.616389336408704; // Mac baseline - Assert.InRange(metrices.MeanSquaredError, expectedMacL2Error - tolerance, expectedMacL2Error + tolerance); + double expectedMacMeanSquaredError = 0.616389336408704; // Mac baseline + Assert.Equal(metrices.MeanSquaredError, expectedMacMeanSquaredError, variationTolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // Windows case - var expectedWindowsL2Error = 0.600329985097577; // Windows baseline - Assert.InRange(metrices.MeanSquaredError, expectedWindowsL2Error - tolerance, expectedWindowsL2Error + tolerance); + double expectedWindowsMeanSquaredError = 0.600329985097577; // Windows baseline + Assert.Equal(metrices.MeanSquaredError, expectedWindowsMeanSquaredError, variationTolerance); } var modelWithValidation = pipeline.Fit(data, testData); @@ -836,5 +846,34 @@ public void InspectMatrixFactorizationModel() for (int i = 0; i < predictions.Count(); ++i) Assert.Equal(predictions[i].Score, valuesAtSecondColumn[i], 3); } + + /// + /// Returns whether or not the current build is CentOS Linux 7. + /// + internal static bool OsIsCentOS7() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + unsafe + { + using (Process process = new Process()) + { + process.StartInfo.FileName = "/bin/bash"; + process.StartInfo.Arguments = "-c \"cat /etc/*-release\""; + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.CreateNoWindow = true; + process.Start(); + + string distro = process.StandardOutput.ReadToEnd().Trim(); + + process.WaitForExit(); + if (distro.Contains("CentOS Linux 7")) + { + return true; + } + } + } + return false; + } } } \ No newline at end of file From a9cd532f4da7b1ab12cab4cf430ba1cf766da626 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 03:18:58 -0700 Subject: [PATCH 3/8] Remove tolerance --- .../TrainerEstimators/MatrixFactorizationTests.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index a88992f0ea..413864f482 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -123,19 +123,18 @@ public void MatrixFactorizationSimpleTrainAndPredict() var metrices = mlContext.Recommendation().Evaluate(prediction, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); // Determine if the selected mean-squared error metric is reasonable on different platforms within the variation tolerance. - int variationTolerance = 7; if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case if (OsIsCentOS7()) { double expectedCentOS7LinuxMeanSquaredError = 0.612732360518435; // CentOS 7 Linux baseline - Assert.Equal(metrices.MeanSquaredError, expectedCentOS7LinuxMeanSquaredError, variationTolerance); + Assert.Equal(metrices.MeanSquaredError, expectedCentOS7LinuxMeanSquaredError); } else { double expectedUbuntuLinuxMeanSquaredError = 0.612726002827395; // Ubuntu Linux baseline - Assert.Equal(metrices.MeanSquaredError, expectedUbuntuLinuxMeanSquaredError, variationTolerance); + Assert.Equal(metrices.MeanSquaredError, expectedUbuntuLinuxMeanSquaredError); } } @@ -143,13 +142,13 @@ public void MatrixFactorizationSimpleTrainAndPredict() { // Mac case double expectedMacMeanSquaredError = 0.616389336408704; // Mac baseline - Assert.Equal(metrices.MeanSquaredError, expectedMacMeanSquaredError, variationTolerance); + Assert.Equal(metrices.MeanSquaredError, expectedMacMeanSquaredError); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // Windows case double expectedWindowsMeanSquaredError = 0.600329985097577; // Windows baseline - Assert.Equal(metrices.MeanSquaredError, expectedWindowsMeanSquaredError, variationTolerance); + Assert.Equal(metrices.MeanSquaredError, expectedWindowsMeanSquaredError); } var modelWithValidation = pipeline.Fit(data, testData); From 733513a4daf1c4fa392796a250d76b74fb6ad807 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 03:32:25 -0700 Subject: [PATCH 4/8] Updated baselines for Ubuntu and Linux, added back tolerance --- .../TrainerEstimators/MatrixFactorizationTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 413864f482..30a5912a2e 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -123,18 +123,19 @@ public void MatrixFactorizationSimpleTrainAndPredict() var metrices = mlContext.Recommendation().Evaluate(prediction, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); // Determine if the selected mean-squared error metric is reasonable on different platforms within the variation tolerance. + int variationTolerance = 7; if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case if (OsIsCentOS7()) { - double expectedCentOS7LinuxMeanSquaredError = 0.612732360518435; // CentOS 7 Linux baseline - Assert.Equal(metrices.MeanSquaredError, expectedCentOS7LinuxMeanSquaredError); + double expectedCentOS7LinuxMeanSquaredError = 0.6127260028273948; // CentOS 7 Linux baseline + Assert.Equal(metrices.MeanSquaredError, expectedCentOS7LinuxMeanSquaredError, variationTolerance); } else { - double expectedUbuntuLinuxMeanSquaredError = 0.612726002827395; // Ubuntu Linux baseline - Assert.Equal(metrices.MeanSquaredError, expectedUbuntuLinuxMeanSquaredError); + double expectedUbuntuLinuxMeanSquaredError = 0.612732360518435; // Ubuntu Linux baseline + Assert.Equal(metrices.MeanSquaredError, expectedUbuntuLinuxMeanSquaredError, variationTolerance); } } @@ -142,13 +143,13 @@ public void MatrixFactorizationSimpleTrainAndPredict() { // Mac case double expectedMacMeanSquaredError = 0.616389336408704; // Mac baseline - Assert.Equal(metrices.MeanSquaredError, expectedMacMeanSquaredError); + Assert.Equal(metrices.MeanSquaredError, expectedMacMeanSquaredError, variationTolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // Windows case double expectedWindowsMeanSquaredError = 0.600329985097577; // Windows baseline - Assert.Equal(metrices.MeanSquaredError, expectedWindowsMeanSquaredError); + Assert.Equal(metrices.MeanSquaredError, expectedWindowsMeanSquaredError, variationTolerance); } var modelWithValidation = pipeline.Fit(data, testData); From f9df598b3fcc1fb82cb49ec110cce0c6a2a0abcd Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 04:19:53 -0700 Subject: [PATCH 5/8] Removed CentOS 7 specific check --- .../MatrixFactorizationTests.cs | 49 +++---------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 30a5912a2e..4e21de8e30 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -123,33 +122,26 @@ public void MatrixFactorizationSimpleTrainAndPredict() var metrices = mlContext.Recommendation().Evaluate(prediction, labelColumnName: labelColumnName, scoreColumnName: scoreColumnName); // Determine if the selected mean-squared error metric is reasonable on different platforms within the variation tolerance. - int variationTolerance = 7; + // Windows and Mac tolerances are set at 1e-7, and Linux tolerance is set at 1e-5. + double windowsAndMacTolerance = Math.Pow(10, -7); + double linuxTolerance = Math.Pow(10, -5); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case - if (OsIsCentOS7()) - { - double expectedCentOS7LinuxMeanSquaredError = 0.6127260028273948; // CentOS 7 Linux baseline - Assert.Equal(metrices.MeanSquaredError, expectedCentOS7LinuxMeanSquaredError, variationTolerance); - } - else - { - double expectedUbuntuLinuxMeanSquaredError = 0.612732360518435; // Ubuntu Linux baseline - Assert.Equal(metrices.MeanSquaredError, expectedUbuntuLinuxMeanSquaredError, variationTolerance); - } - + double expectedLinuxMacMeanSquaredError = 0.6127260028273948; // Linux baseline + Assert.InRange(metrices.MeanSquaredError, expectedLinuxMacMeanSquaredError - linuxTolerance, expectedLinuxMacMeanSquaredError + linuxTolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { // Mac case double expectedMacMeanSquaredError = 0.616389336408704; // Mac baseline - Assert.Equal(metrices.MeanSquaredError, expectedMacMeanSquaredError, variationTolerance); + Assert.InRange(metrices.MeanSquaredError, expectedMacMeanSquaredError - windowsAndMacTolerance, expectedMacMeanSquaredError + windowsAndMacTolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // Windows case double expectedWindowsMeanSquaredError = 0.600329985097577; // Windows baseline - Assert.Equal(metrices.MeanSquaredError, expectedWindowsMeanSquaredError, variationTolerance); + Assert.InRange(metrices.MeanSquaredError, expectedWindowsMeanSquaredError - windowsAndMacTolerance, expectedWindowsMeanSquaredError + windowsAndMacTolerance); } var modelWithValidation = pipeline.Fit(data, testData); @@ -847,33 +839,6 @@ public void InspectMatrixFactorizationModel() Assert.Equal(predictions[i].Score, valuesAtSecondColumn[i], 3); } - /// - /// Returns whether or not the current build is CentOS Linux 7. - /// - internal static bool OsIsCentOS7() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - unsafe - { - using (Process process = new Process()) - { - process.StartInfo.FileName = "/bin/bash"; - process.StartInfo.Arguments = "-c \"cat /etc/*-release\""; - process.StartInfo.UseShellExecute = false; - process.StartInfo.RedirectStandardOutput = true; - process.StartInfo.CreateNoWindow = true; - process.Start(); - - string distro = process.StandardOutput.ReadToEnd().Trim(); - - process.WaitForExit(); - if (distro.Contains("CentOS Linux 7")) - { - return true; - } - } - } - return false; } } } \ No newline at end of file From 67787c891553fb45da4f03f960dfe4123935df77 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 04:37:13 -0700 Subject: [PATCH 6/8] Removed extra braces and space --- .../TrainerEstimators/MatrixFactorizationTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 4e21de8e30..7d9f97a3f6 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -838,7 +838,5 @@ public void InspectMatrixFactorizationModel() for (int i = 0; i < predictions.Count(); ++i) Assert.Equal(predictions[i].Score, valuesAtSecondColumn[i], 3); } - - } } } \ No newline at end of file From d119545058a9d0552e2699ea94242fde8a438112 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 12:02:31 -0700 Subject: [PATCH 7/8] Updated name of expectedLinuxMeanSquaredError --- .../TrainerEstimators/MatrixFactorizationTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index 7d9f97a3f6..f87c7046a3 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -128,8 +128,8 @@ public void MatrixFactorizationSimpleTrainAndPredict() if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { // Linux case - double expectedLinuxMacMeanSquaredError = 0.6127260028273948; // Linux baseline - Assert.InRange(metrices.MeanSquaredError, expectedLinuxMacMeanSquaredError - linuxTolerance, expectedLinuxMacMeanSquaredError + linuxTolerance); + double expectedLinuxMeanSquaredError = 0.6127260028273948; // Linux baseline + Assert.InRange(metrices.MeanSquaredError, expectedLinuxMeanSquaredError - linuxTolerance, expectedLinuxMeanSquaredError + linuxTolerance); } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { From 995df7f8939b6df3db8e1577a17d29c2c7760ce3 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 13 May 2020 15:07:48 -0700 Subject: [PATCH 8/8] Added explanation of varying but consistent metrics --- .../TrainerEstimators/MatrixFactorizationTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs index f87c7046a3..ab6e09ea0a 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/MatrixFactorizationTests.cs @@ -123,6 +123,9 @@ public void MatrixFactorizationSimpleTrainAndPredict() // Determine if the selected mean-squared error metric is reasonable on different platforms within the variation tolerance. // Windows and Mac tolerances are set at 1e-7, and Linux tolerance is set at 1e-5. + // Here, each build OS has a different MSE baseline metric. While these metrics differ between builds, each build is expected to + // produce the same metric. This is because of minor build differences and varying implementations of sub-functions, such as random + // variables that are first obtained with the default random numger generator in libMF C++ libraries. double windowsAndMacTolerance = Math.Pow(10, -7); double linuxTolerance = Math.Pow(10, -5); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))