Initial implementation of LpNormalizer.

src/python/nimbusml.pyproj

@@ -118,6 +118,7 @@
     <Compile Include="nimbusml\examples\examples_from_dataframe\Filter_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\GamBinaryClassifier_infert_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\GamRegressor_airquality_df.py" />
+    <Compile Include="nimbusml\examples\examples_from_dataframe\LpScaler_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\GlobalContrastRowScaler_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\Handler_df.py" />
     <Compile Include="nimbusml\examples\examples_from_dataframe\IidChangePointDetector_df.py" />
@@ -170,6 +171,7 @@
     <Compile Include="nimbusml\examples\Filter.py" />
     <Compile Include="nimbusml\examples\GamBinaryClassifier.py" />
     <Compile Include="nimbusml\examples\GamRegressor.py" />
+    <Compile Include="nimbusml\examples\LpScaler.py" />
     <Compile Include="nimbusml\examples\GlobalContrastRowScaler.py" />
     <Compile Include="nimbusml\examples\Handler.py" />
     <Compile Include="nimbusml\examples\Hinge.py" />
@@ -296,6 +298,7 @@
     <Compile Include="nimbusml\internal\core\preprocessing\datasettransformer.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\filter\skipfilter.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\filter\takefilter.py" />
+    <Compile Include="nimbusml\internal\core\preprocessing\normalization\lpscaler.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\schema\columnduplicator.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\schema\columndropper.py" />
     <Compile Include="nimbusml\internal\core\preprocessing\tensorflowscorer.py" />
@@ -626,6 +629,7 @@
     <Compile Include="nimbusml\preprocessing\normalization\binner.py" />
     <Compile Include="nimbusml\preprocessing\normalization\globalcontrastrowscaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\logmeanvariancescaler.py" />
+    <Compile Include="nimbusml\preprocessing\normalization\lpscaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\meanvariancescaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\minmaxscaler.py" />
     <Compile Include="nimbusml\preprocessing\normalization\__init__.py" />
@@ -667,6 +671,7 @@
     <Compile Include="nimbusml\tests\linear_model\test_linearsvmbinaryclassifier.py" />
     <Compile Include="nimbusml\tests\pipeline\test_pipeline_combining.py" />
     <Compile Include="nimbusml\tests\pipeline\test_pipeline_subclassing.py" />
+    <Compile Include="nimbusml\tests\preprocessing\normalization\test_lpscaler.py" />
     <Compile Include="nimbusml\tests\preprocessing\normalization\test_meanvariancescaler.py" />
     <Compile Include="nimbusml\tests\preprocessing\test_datasettransformer.py" />
     <Compile Include="nimbusml\tests\test_csr_matrix_output.py" />

src/python/nimbusml/examples/LpScaler.py

@@ -0,0 +1,47 @@
+###############################################################################
+# LpScaler
+import numpy
+from nimbusml import FileDataStream
+from nimbusml.datasets import get_dataset
+from nimbusml.preprocessing.normalization import LpScaler
+
+path = get_dataset('infert').as_filepath()
+data = FileDataStream.read_csv(
+    path,
+    sep=',',
+    numeric_dtype=numpy.float32,
+    collapse=True)
+
+print(data.head())
+
+# row_num education  age.age  age.parity  age.induced  age.case  age.spontaneous  age.stratum  age.pooled.stratum
+#     1.0    0-5yrs     26.0         6.0          1.0       1.0              2.0          1.0                 3.0
+#     2.0    0-5yrs     42.0         1.0          1.0       1.0              0.0          2.0                 1.0
+#     3.0    0-5yrs     39.0         6.0          2.0       1.0              0.0          3.0                 4.0
+#     4.0    0-5yrs     34.0         4.0          2.0       1.0              0.0          4.0                 2.0
+#     5.0   6-11yrs     35.0         3.0          1.0       1.0              1.0          5.0                32.0
+
+xf = LpScaler(columns={'norm': 'age'})
+features = xf.fit_transform(data)
+
+print_opts = {
+    'index': False,
+    'justify': 'left',
+    'columns': [
+        'norm.age',
+        'norm.parity',
+        'norm.induced',
+        'norm.case',
+        'norm.spontaneous',
+        'norm.stratum',
+        'norm.pooled.stratum'
+    ]
+}
+print('LpScaler\n', features.head().to_string(**print_opts))
+
+# norm.age  norm.parity  norm.induced  norm.case  norm.spontaneous  norm.stratum  norm.pooled.stratum
+# 0.963624  0.222375     0.037062      0.037062   0.074125          0.037062      0.111187
+# 0.997740  0.023756     0.023756      0.023756   0.000000          0.047511      0.023756
+# 0.978985  0.150613     0.050204      0.025102   0.000000          0.075307      0.100409
+# 0.982725  0.115615     0.057807      0.028904   0.000000          0.115615      0.057807
+# 0.732032  0.062746     0.020915      0.020915   0.020915          0.104576      0.669286

src/python/nimbusml/examples/examples_from_dataframe/LpScaler_df.py

@@ -0,0 +1,48 @@
+###############################################################################
+# LpScaler
+import numpy as np
+import pandas as pd
+from nimbusml import Pipeline
+from nimbusml.preprocessing.normalization import LpScaler
+from nimbusml.preprocessing.schema import ColumnConcatenator
+
+in_df = pd.DataFrame(
+    data=dict(
+        Sepal_Length=[2.5, 1, 2.1, 1.0],
+        Sepal_Width=[.75, .9, .8, .76],
+        Petal_Length=[0, 2.5, 2.6, 2.4],
+        Species=["setosa", "viginica", "setosa", 'versicolor']))
+
+in_df.iloc[:, 0:3] = in_df.iloc[:, 0:3].astype(np.float32)
+
+concat = ColumnConcatenator() << {
+    'cat': [ 'Sepal_Length', 'Sepal_Width', 'Petal_Length']
+}
+
+# Normalize the input values by rescaling them to unit norm (L2, L1 or LInf).
+# Performs the following operation on a vector X: Y = (X - M) / D, where M is
+# mean and D is either L2 norm, L1 norm or LInf norm.
+normed = LpScaler() << {'norm': 'cat'}
+
+pipeline = Pipeline([concat, normed])
+out_df = pipeline.fit_transform(in_df)
+
+print_opts = {
+    'index': False,
+    'justify': 'left',
+    'columns': [
+        'Sepal_Length',
+        'Sepal_Width',
+        'Petal_Length',
+        'norm.Sepal_Length',
+        'norm.Sepal_Width',
+        'norm.Petal_Length'
+    ]
+}
+print('LpScaler\n', out_df.to_string(**print_opts))
+
+# Sepal_Length  Sepal_Width  Petal_Length  norm.Sepal_Length  norm.Sepal_Width  norm.Petal_Length
+# 2.5           0.75         0.0           0.957826           0.287348          0.000000
+# 1.0           0.90         2.5           0.352235           0.317011          0.880587
+# 2.1           0.80         2.6           0.611075           0.232790          0.756569
+# 1.0           0.76         2.4           0.369167           0.280567          0.886001

src/python/nimbusml/internal/core/preprocessing/normalization/lpscaler.py

@@ -0,0 +1,93 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------------------------
+# - Generated by tools/entrypoint_compiler.py: do not edit by hand
+"""
+LpScaler
+"""
+
+__all__ = ["LpScaler"]
+
+
+from ....entrypoints.transforms_lpnormalizer import transforms_lpnormalizer
+from ....utils.utils import trace
+from ...base_pipeline_item import BasePipelineItem, DefaultSignature
+
+
+class LpScaler(BasePipelineItem, DefaultSignature):
+    """
+    **Description**
+        Normalize vectors (rows) individually by rescaling them to unit norm (L2, L1 or LInf). Performs the following operation on a vector X: Y = (X - M) / D, where M is mean and D is either L2 norm, L1 norm or LInf norm.
+
+    :param norm: The norm to use to normalize each sample.
+
+    :param sub_mean: Subtract mean from each value before normalizing.
+
+    :param params: Additional arguments sent to compute engine.
+
+    """
+
+    @trace
+    def __init__(
+            self,
+            norm='L2',
+            sub_mean=False,
+            **params):
+        BasePipelineItem.__init__(
+            self, type='transform', **params)
+
+        self.norm = norm
+        self.sub_mean = sub_mean
+
+    @property
+    def _entrypoint(self):
+        return transforms_lpnormalizer
+
+    @trace
+    def _get_node(self, **all_args):
+
+        input_columns = self.input
+        if input_columns is None and 'input' in all_args:
+            input_columns = all_args['input']
+        if 'input' in all_args:
+            all_args.pop('input')
+
+        output_columns = self.output
+        if output_columns is None and 'output' in all_args:
+            output_columns = all_args['output']
+        if 'output' in all_args:
+            all_args.pop('output')
+
+        # validate input
+        if input_columns is None:
+            raise ValueError(
+                "'None' input passed when it cannot be none.")
+
+        if not isinstance(input_columns, list):
+            raise ValueError(
+                "input has to be a list of strings, instead got %s" %
+                type(input_columns))
+
+        # validate output
+        if output_columns is None:
+            output_columns = input_columns
+
+        if not isinstance(output_columns, list):
+            raise ValueError(
+                "output has to be a list of strings, instead got %s" %
+                type(output_columns))
+
+        algo_args = dict(
+            column=[
+                dict(
+                    Source=i,
+                    Name=o) for i,
+                o in zip(
+                    input_columns,
+                    output_columns)] if input_columns else None,
+            norm=self.norm,
+            sub_mean=self.sub_mean)
+
+        all_args.update(algo_args)
+        return self._entrypoint(**all_args)

src/python/nimbusml/preprocessing/normalization/__init__.py

@@ -1,13 +1,15 @@
 from .binner import Binner
 from .globalcontrastrowscaler import GlobalContrastRowScaler
 from .logmeanvariancescaler import LogMeanVarianceScaler
+from .lpscaler import LpScaler
 from .meanvariancescaler import MeanVarianceScaler
 from .minmaxscaler import MinMaxScaler
 
 __all__ = [
     'Binner',
     'GlobalContrastRowScaler',
     'LogMeanVarianceScaler',
+    'LpScaler',
     'MeanVarianceScaler',
-    'MinMaxScaler',
+    'MinMaxScaler'
 ]

src/python/nimbusml/preprocessing/normalization/lpscaler.py

@@ -0,0 +1,68 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------------------------
+# - Generated by tools/entrypoint_compiler.py: do not edit by hand
+"""
+LpScaler
+"""
+
+__all__ = ["LpScaler"]
+
+
+from sklearn.base import TransformerMixin
+
+from ...base_transform import BaseTransform
+from ...internal.core.preprocessing.normalization.lpscaler import \
+    LpScaler as core
+from ...internal.utils.utils import trace
+
+
+class LpScaler(core, BaseTransform, TransformerMixin):
+    """
+    **Description**
+        Normalize vectors (rows) individually by rescaling them to unit norm (L2, L1 or LInf). Performs the following operation on a vector X: Y = (X - M) / D, where M is mean and D is either L2 norm, L1 norm or LInf norm.
+
+    :param columns: see `Columns </nimbusml/concepts/columns>`_.
+
+    :param norm: The norm to use to normalize each sample.
+
+    :param sub_mean: Subtract mean from each value before normalizing.
+
+    :param params: Additional arguments sent to compute engine.
+
+    """
+
+    @trace
+    def __init__(
+            self,
+            norm='L2',
+            sub_mean=False,
+            columns=None,
+            **params):
+
+        if columns:
+            params['columns'] = columns
+        BaseTransform.__init__(self, **params)
+        core.__init__(
+            self,
+            norm=norm,
+            sub_mean=sub_mean,
+            **params)
+        self._columns = columns
+
+    def get_params(self, deep=False):
+        """
+        Get the parameters for this operator.
+        """
+        return core.get_params(self)
+
+    def _nodes_with_presteps(self):
+        """
+        Inserts preprocessing before this one.
+        """
+        from ..schema import TypeConverter
+        return [
+            TypeConverter(
+                result_type='R4')._steal_io(self),
+            self]

src/python/nimbusml/tests/preprocessing/normalization/test_lpscaler.py

@@ -0,0 +1,70 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------------------------
+
+import unittest
+
+import numpy as np
+import pandas as pd
+from nimbusml import Pipeline
+from nimbusml.preprocessing.normalization import LpScaler
+from nimbusml.preprocessing.schema import ColumnConcatenator
+from sklearn.utils.testing import assert_greater, assert_less
+
+
+class TestLpScaler(unittest.TestCase):
+
+    def test_lpscaler(self):
+        in_df = pd.DataFrame(
+            data=dict(
+                Sepal_Length=[2.5, 1, 2.1, 1.0],
+                Sepal_Width=[.75, .9, .8, .76],
+                Petal_Length=[0, 2.5, 2.6, 2.4],
+                Species=["setosa", "viginica", "setosa", 'versicolor']))
+
+        in_df.iloc[:, 0:3] = in_df.iloc[:, 0:3].astype(np.float32)
+
+        src_cols = ['Sepal_Length', 'Sepal_Width', 'Petal_Length']
+
+        pipeline = Pipeline([
+            ColumnConcatenator() << {'concat': src_cols},
+            LpScaler() << {'norm': 'concat'}
+        ])
+        out_df = pipeline.fit_transform(in_df)
+
+        cols = ['concat.' + s for s in src_cols]
+        cols.extend(['norm.' + s for s in src_cols])
+        sum = out_df[cols].sum().sum()
+        sum_range = (23.24, 23.25)
+        assert_greater(sum, sum_range[0], "sum should be greater than %s" % sum_range[0])
+        assert_less(sum, sum_range[1], "sum should be less than %s" % sum_range[1])
+
+    def test_lpscaler_automatically_converts_to_single(self):
+        in_df = pd.DataFrame(
+            data=dict(
+                Sepal_Length=[2.5, 1, 2.1, 1.0],
+                Sepal_Width=[.75, .9, .8, .76],
+                Petal_Length=[0, 2.5, 2.6, 2.4],
+                Species=["setosa", "viginica", "setosa", 'versicolor']))
+
+        in_df.iloc[:, 0:3] = in_df.iloc[:, 0:3].astype(np.float64)
+
+        src_cols = ['Sepal_Length', 'Sepal_Width', 'Petal_Length']
+
+        pipeline = Pipeline([
+            ColumnConcatenator() << {'concat': src_cols},
+            LpScaler() << {'norm': 'concat'}
+        ])
+        out_df = pipeline.fit_transform(in_df)
+
+        cols = ['concat.' + s for s in src_cols]
+        cols.extend(['norm.' + s for s in src_cols])
+        sum = out_df[cols].sum().sum()
+        sum_range = (23.24, 23.25)
+        assert_greater(sum, sum_range[0], "sum should be greater than %s" % sum_range[0])
+        assert_less(sum, sum_range[1], "sum should be less than %s" % sum_range[1])
+
+
+if __name__ == '__main__':
+    unittest.main()

src/python/tests/test_estimator_checks.py

@@ -160,8 +160,7 @@
     'PixelExtractor, Loader, Resizer, \
                         GlobalContrastRowScaler, PcaTransformer, '
     'ColumnConcatenator, Sentiment, CharTokenizer, LightLda, '
-    'NGramFeaturizer, \
-                        WordEmbedding',
+    'NGramFeaturizer, WordEmbedding, LpScaler',
     'check_transformer_data_not_an_array, check_pipeline_consistency, '
     'check_fit2d_1feature, check_estimators_fit_returns_self,\
                        check_fit2d_1sample, '