From b232ffcdab3f3416a080afce6649b89f0080fa7d Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 15 Jan 2020 12:10:56 -0800 Subject: [PATCH 1/3] added in support for System.DateTime type for the DateTimeTransformer --- .../DateTimeTransformer.cs | 132 ++++++++++-------- .../Transformers/DateTimeTransformerTests.cs | 107 ++++++++++++-- 2 files changed, 171 insertions(+), 68 deletions(-) diff --git a/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs b/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs index 1f100e49a5..80a78412d7 100644 --- a/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs +++ b/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs @@ -148,7 +148,7 @@ internal DateTimeEstimator(IHostEnvironment env, Options options) public DateTimeTransformer Fit(IDataView input) { - return new DateTimeTransformer(_host, _options.Source, _options.Prefix, _options.Country); + return new DateTimeTransformer(_host, _options.Source, _options.Prefix, _options.Country, input); } public SchemaShape GetOutputSchema(SchemaShape inputSchema) @@ -246,14 +246,22 @@ public sealed class DateTimeTransformer : RowToRowTransformerBase, IDisposable internal const string LoadName = "DateTimeTransform"; internal const string LoaderSignature = "DateTimeTransform"; private LongTypedColumn _column; + private DataViewSchema _schema; #endregion - internal DateTimeTransformer(IHostEnvironment host, string inputColumnName, string columnPrefix, DateTimeEstimator.HolidayList country) : + internal DateTimeTransformer(IHostEnvironment host, string inputColumnName, string columnPrefix, DateTimeEstimator.HolidayList country, IDataView input) : base(host.Register(nameof(DateTimeTransformer))) { host.Check(!CommonExtensions.OsIsCentOS7(), "CentOS7 is not supported"); + _schema = input.Schema; + if (_schema[inputColumnName].Type.RawType != typeof(long) && + _schema[inputColumnName].Type.RawType != typeof(DateTime)) + { + throw new Exception($"Unsupported type {_schema[inputColumnName].Type.RawType} for input column ${inputColumnName}. Only long and System.DateTime are supported"); + } + _column = new LongTypedColumn(inputColumnName, columnPrefix); _column.CreateTransformerFromEstimator(country); } @@ -443,9 +451,11 @@ private static unsafe string GetStringFromPointer(ref ReadOnlySpan rawData }; -#endregion + #endregion + + #region ColumnInfo -#region BaseClass + #region BaseClass internal delegate bool DestroyCppTransformerEstimator(IntPtr estimator, out IntPtr errorHandle); internal delegate bool DestroyTransformerSaveData(IntPtr buffer, IntPtr bufferSize, out IntPtr errorHandle); @@ -455,11 +465,17 @@ internal abstract class TypedColumn : IDisposable { internal readonly string Source; internal readonly string Prefix; + internal readonly int IntPtrSize; + internal readonly int StructSize; internal TypedColumn(string source, string prefix) { Source = source; Prefix = prefix; + IntPtrSize = IntPtr.Size; + + // The native struct is 25 bytes + 8 size_t. + StructSize = 25 + (IntPtrSize * 8); } internal abstract void CreateTransformerFromEstimator(DateTimeEstimator.HolidayList country); @@ -542,22 +558,17 @@ internal TypedColumn(string source, string prefix) : } -#endregion + #endregion BaseClass -#region DateTimeTypedColumn + #region LongTypedColumn internal sealed class LongTypedColumn : TypedColumn { private TransformerEstimatorSafeHandle _transformerHandler; - private readonly int _intPtrSize; - private readonly int _structSize; + internal LongTypedColumn(string source, string prefix) : base(source, prefix) { - _intPtrSize = IntPtr.Size; - - // The native struct is 25 bytes + 8 size_t. - _structSize = 25 + (_intPtrSize * 8); } [DllImport("Featurizers", EntryPoint = "DateTimeFeaturizer_CreateEstimator"), SuppressUnmanagedCodeSecurity] @@ -601,10 +612,9 @@ internal override TimePoint Transform(long input) using (var handler = new TransformedDataSafeHandle(output, DestroyTransformedDataNative)) { - // 29 plus size. unsafe { - return new TimePoint(new ReadOnlySpan(output.ToPointer(), _structSize), _intPtrSize); + return new TimePoint(new ReadOnlySpan(output.ToPointer(), StructSize), IntPtrSize); } } } @@ -633,7 +643,9 @@ private protected override bool CreateTransformerSaveDataHelper(out IntPtr buffe CreateTransformerSaveDataNative(_transformerHandler, out buffer, out bufferSize, out errorHandle); } -#endregion + #endregion LongTypedColumn + + #endregion ColumnInfo private sealed class Mapper : MapperBase { @@ -643,6 +655,7 @@ private sealed class Mapper : MapperBase private readonly DateTimeTransformer _parent; private ConcurrentDictionary _cache; private ConcurrentQueue _oldestKeys; + private readonly DateTime _unixEpoc; #endregion @@ -650,6 +663,7 @@ public Mapper(DateTimeTransformer parent, DataViewSchema inputSchema) : base(parent.Host.Register(nameof(Mapper)), inputSchema, parent) { _parent = parent; + _unixEpoc = new DateTime(1970, 1, 1); _cache = new ConcurrentDictionary(); _oldestKeys = new ConcurrentQueue(); } @@ -667,74 +681,68 @@ protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() return columns.ToArray(); } - private Delegate MakeGetter(DataViewRow input, int iinfo) + private Delegate MakeGetter(DataViewRow input, int iinfo) { - var getter = input.GetGetter(input.Schema[_parent._column.Source]); - ValueGetter result = (ref T dst) => + var getter = input.GetGetter(input.Schema[_parent._column.Source]); + + ValueGetter result = (ref TTransformed dst) => { - long dateTime = default; - getter(ref dateTime); + TInput dateTimeColumn = default; + getter(ref dateTimeColumn); - if (!_cache.TryGetValue(dateTime, out TimePoint timePoint)) - { - _cache[dateTime] = _parent._column.Transform(dateTime); - _oldestKeys.Enqueue(dateTime); - timePoint = _cache[dateTime]; - - // If more than 100 cached items, remove 20 - if (_cache.Count > 100) - { - for (int i = 0; i < 20; i++) - { - long key; - while (!_oldestKeys.TryDequeue(out key)) { } - while (!_cache.TryRemove(key, out TimePoint removedValue)) { } - } - } - } + long dateTime; + + // Type already in Posix format + if(typeof(TInput) == typeof(long)) + dateTime = (long)(object)dateTimeColumn; + // System.DateTime + else + dateTime = ((DateTime)(object)dateTimeColumn).Subtract(_unixEpoc).Ticks / TimeSpan.TicksPerSecond; + + var timePoint = _parent._column.Transform(dateTime); if (iinfo == 0) - dst = (T)Convert.ChangeType(timePoint.Year, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.Year, typeof(TTransformed)); else if (iinfo == 1) - dst = (T)Convert.ChangeType(timePoint.Month, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.Month, typeof(TTransformed)); else if (iinfo == 2) - dst = (T)Convert.ChangeType(timePoint.Day, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.Day, typeof(TTransformed)); else if (iinfo == 3) - dst = (T)Convert.ChangeType(timePoint.Hour, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.Hour, typeof(TTransformed)); else if (iinfo == 4) - dst = (T)Convert.ChangeType(timePoint.Minute, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.Minute, typeof(TTransformed)); else if (iinfo == 5) - dst = (T)Convert.ChangeType(timePoint.Second, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.Second, typeof(TTransformed)); else if (iinfo == 6) - dst = (T)Convert.ChangeType(timePoint.AmPm, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.AmPm, typeof(TTransformed)); else if (iinfo == 7) - dst = (T)Convert.ChangeType(timePoint.Hour12, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.Hour12, typeof(TTransformed)); else if (iinfo == 8) - dst = (T)Convert.ChangeType(timePoint.DayOfWeek, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.DayOfWeek, typeof(TTransformed)); else if (iinfo == 9) - dst = (T)Convert.ChangeType(timePoint.DayOfQuarter, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.DayOfQuarter, typeof(TTransformed)); else if (iinfo == 10) - dst = (T)Convert.ChangeType(timePoint.DayOfYear, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.DayOfYear, typeof(TTransformed)); else if (iinfo == 11) - dst = (T)Convert.ChangeType(timePoint.WeekOfMonth, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.WeekOfMonth, typeof(TTransformed)); else if (iinfo == 12) - dst = (T)Convert.ChangeType(timePoint.QuarterOfYear, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.QuarterOfYear, typeof(TTransformed)); else if (iinfo == 13) - dst = (T)Convert.ChangeType(timePoint.HalfOfYear, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.HalfOfYear, typeof(TTransformed)); else if (iinfo == 14) - dst = (T)Convert.ChangeType(timePoint.WeekIso, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.WeekIso, typeof(TTransformed)); else if (iinfo == 15) - dst = (T)Convert.ChangeType(timePoint.YearIso, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.YearIso, typeof(TTransformed)); else if (iinfo == 16) - dst = (T)Convert.ChangeType(timePoint.MonthLabel.AsMemory(), typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.MonthLabel.AsMemory(), typeof(TTransformed)); else if (iinfo == 17) - dst = (T)Convert.ChangeType(timePoint.AmPmLabel.AsMemory(), typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.AmPmLabel.AsMemory(), typeof(TTransformed)); else if (iinfo == 18) - dst = (T)Convert.ChangeType(timePoint.DayOfWeekLabel.AsMemory(), typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.DayOfWeekLabel.AsMemory(), typeof(TTransformed)); else if (iinfo == 19) - dst = (T)Convert.ChangeType(timePoint.HolidayName.AsMemory(), typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.HolidayName.AsMemory(), typeof(TTransformed)); else - dst = (T)Convert.ChangeType(timePoint.IsPaidTimeOff, typeof(T)); + dst = (TTransformed)Convert.ChangeType(timePoint.IsPaidTimeOff, typeof(TTransformed)); }; return result; @@ -745,7 +753,13 @@ protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func, ((DateTimeEstimator.ColumnsProduced)iinfo + 1).GetRawColumnType(), input, iinfo); + // If input type is long, Posix time + if (input.Schema[_parent._column.Source].Type.RawType == typeof(long)) + return Utils.MarshalInvoke(MakeGetter, new Type[] { typeof(long), ((DateTimeEstimator.ColumnsProduced)iinfo + 1).GetRawColumnType() }, input, iinfo); + // If input types is System.DateTime + else + return Utils.MarshalInvoke(MakeGetter, new Type[] { typeof(DateTime), ((DateTimeEstimator.ColumnsProduced)iinfo + 1).GetRawColumnType() }, input, iinfo); + } private protected override Func GetDependenciesCore(Func activeOutput) diff --git a/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs b/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs index 4b91b30586..d28b844799 100644 --- a/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs @@ -23,7 +23,7 @@ private class DateTimeInput public long date; } - [NotCentOS7FactAttribute] + [NotCentOS7Fact] public void CorrectNumberOfColumnsAndSchema() { MLContext mlContext = new MLContext(1); @@ -90,8 +90,8 @@ public void CorrectNumberOfColumnsAndSchema() Done(); } - [NotCentOS7FactAttribute] - public void CanUseDateFromColumn() + [NotCentOS7Fact] + public void CanUseDateFromColumnLongType() { // Future Date - 2025 June 30 MLContext mlContext = new MLContext(1); @@ -133,7 +133,50 @@ public void CanUseDateFromColumn() Done(); } - [NotCentOS7FactAttribute] + [NotCentOS7Fact] + public void CanUseDateFromColumnDateTimeType() + { + // Future Date - 2025 June 30 + MLContext mlContext = new MLContext(1); + var dataList = new[] { new { date = new DateTime(2025,6,30)} }; + var data = mlContext.Data.LoadFromEnumerable(dataList); + + // Build the pipeline, fit, and transform it. + var pipeline = mlContext.Transforms.FeaturizeDateTime("date", "DTC"); + var model = pipeline.Fit(data); + var output = model.Transform(data); + + // Get the data from the first row and make sure it matches expected + var row = output.Preview(1).RowView[0].Values; + + // Assert the data from the first row is what we expect + Assert.Equal(2025, row[1].Value); // Year + Assert.Equal((byte)6, row[2].Value); // Month + Assert.Equal((byte)30, row[3].Value); // Day + Assert.Equal((byte)0, row[4].Value); // Hour + Assert.Equal((byte)0, row[5].Value); // Minute + Assert.Equal((byte)0, row[6].Value); // Second + Assert.Equal((byte)0, row[7].Value); // AmPm + Assert.Equal((byte)0, row[8].Value); // Hour12 + Assert.Equal((byte)1, row[9].Value); // DayOfWeek + Assert.Equal((byte)91, row[10].Value); // DayOfQuarter + Assert.Equal((ushort)180, row[11].Value); // DayOfYear + Assert.Equal((ushort)4, row[12].Value); // WeekOfMonth + Assert.Equal((byte)2, row[13].Value); // QuarterOfYear + Assert.Equal((byte)1, row[14].Value); // HalfOfYear + Assert.Equal((byte)27, row[15].Value); // WeekIso + Assert.Equal(2025, row[16].Value); // YearIso + Assert.Equal("June", row[17].Value.ToString()); // MonthLabel + Assert.Equal("am", row[18].Value.ToString()); // AmPmLabel + Assert.Equal("Monday", row[19].Value.ToString()); // DayOfWeekLabel + Assert.Equal("", row[20].Value.ToString()); // HolidayName + Assert.Equal((byte)0, row[21].Value); // IsPaidTimeOff + + TestEstimatorCore(pipeline, data); + Done(); + } + + [NotCentOS7Fact] public void HolidayTest() { // Future Date - 2025 June 30 @@ -157,15 +200,61 @@ public void HolidayTest() Done(); } - [NotCentOS7FactAttribute] - public void ManyRowsTest() + [NotCentOS7Fact] + public void ManyRowsTestLongType() { // Future Date - 2025 June 30 MLContext mlContext = new MLContext(1); var dataList = new[] { new DateTimeInput() { date = 1751241600 }, new DateTimeInput() { date = 1751241600 }, new DateTimeInput() { date = 12341 }, new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 1234 }, new DateTimeInput() { date = 1751241600 }, - new DateTimeInput() { date = 1751241600 }, new DateTimeInput() { date = 12341 }, - new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 1234 }}; + new DateTimeInput() { date = 1751241600 }, new DateTimeInput() { date = 12341 }, new DateTimeInput() { date = 1234 }, new DateTimeInput() { date = 1234 }, + new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 1234 }, new DateTimeInput() { date = 1234 }}; + + var data = mlContext.Data.LoadFromEnumerable(dataList); + + // Build the pipeline, fit, and transform it. + var pipeline = mlContext.Transforms.FeaturizeDateTime("date", "DTC"); + var model = pipeline.Fit(data); + var output = model.Transform(data); + + // Get the data from the first row and make sure it matches expected + var row = output.Preview().RowView[0].Values; + + // Assert the data from the first row is what we expect + Assert.Equal(2025, row[1].Value); // Year + Assert.Equal((byte)6, row[2].Value); // Month + Assert.Equal((byte)30, row[3].Value); // Day + Assert.Equal((byte)0, row[4].Value); // Hour + Assert.Equal((byte)0, row[5].Value); // Minute + Assert.Equal((byte)0, row[6].Value); // Second + Assert.Equal((byte)0, row[7].Value); // AmPm + Assert.Equal((byte)0, row[8].Value); // Hour12 + Assert.Equal((byte)1, row[9].Value); // DayOfWeek + Assert.Equal((byte)91, row[10].Value); // DayOfQuarter + Assert.Equal((ushort)180, row[11].Value); // DayOfYear + Assert.Equal((ushort)4, row[12].Value); // WeekOfMonth + Assert.Equal((byte)2, row[13].Value); // QuarterOfYear + Assert.Equal((byte)1, row[14].Value); // HalfOfYear + Assert.Equal((byte)27, row[15].Value); // WeekIso + Assert.Equal(2025, row[16].Value); // YearIso + Assert.Equal("June", row[17].Value.ToString()); // MonthLabel + Assert.Equal("am", row[18].Value.ToString()); // AmPmLabel + Assert.Equal("Monday", row[19].Value.ToString()); // DayOfWeekLabel + Assert.Equal("", row[20].Value.ToString()); // HolidayName + Assert.Equal((byte)0, row[21].Value); // IsPaidTimeOff + + TestEstimatorCore(pipeline, data); + Done(); + } + + [NotCentOS7Fact] + public void ManyRowsTestDateTimeType() + { + // Future Date - 2025 June 30 + MLContext mlContext = new MLContext(1); + var dataList = new[] { new { date = new DateTime(2025, 6, 30) }, new { date = new DateTime(2025, 6, 30) }, + new { date = DateTime.Now }, new { date = DateTime.UtcNow }, + new { date = DateTime.Now }, new { date = DateTime.UtcNow } }; var data = mlContext.Data.LoadFromEnumerable(dataList); @@ -204,7 +293,7 @@ public void ManyRowsTest() Done(); } - [NotCentOS7FactAttribute] + [NotCentOS7Fact] public void EntryPointTest() { // Future Date - 2025 June 30 From d49d90bf6624089fceec69ba32b658d7c26f2cb6 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 17 Jan 2020 14:37:50 -0800 Subject: [PATCH 2/3] updates from PR comments --- .../DateTimeTransformer.cs | 147 ++++++++++-------- .../Transformers/DateTimeTransformerTests.cs | 12 +- 2 files changed, 87 insertions(+), 72 deletions(-) diff --git a/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs b/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs index 80a78412d7..d0748e3fce 100644 --- a/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs +++ b/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs @@ -148,7 +148,7 @@ internal DateTimeEstimator(IHostEnvironment env, Options options) public DateTimeTransformer Fit(IDataView input) { - return new DateTimeTransformer(_host, _options.Source, _options.Prefix, _options.Country, input); + return new DateTimeTransformer(_host, _options.Source, _options.Prefix, _options.Country, input.Schema); } public SchemaShape GetOutputSchema(SchemaShape inputSchema) @@ -250,12 +250,12 @@ public sealed class DateTimeTransformer : RowToRowTransformerBase, IDisposable #endregion - internal DateTimeTransformer(IHostEnvironment host, string inputColumnName, string columnPrefix, DateTimeEstimator.HolidayList country, IDataView input) : + internal DateTimeTransformer(IHostEnvironment host, string inputColumnName, string columnPrefix, DateTimeEstimator.HolidayList country, DataViewSchema schema) : base(host.Register(nameof(DateTimeTransformer))) { host.Check(!CommonExtensions.OsIsCentOS7(), "CentOS7 is not supported"); - _schema = input.Schema; + _schema = schema; if (_schema[inputColumnName].Type.RawType != typeof(long) && _schema[inputColumnName].Type.RawType != typeof(DateTime)) { @@ -650,20 +650,19 @@ private protected override bool CreateTransformerSaveDataHelper(out IntPtr buffe private sealed class Mapper : MapperBase { -#region Class data members + #region Class data members + private static readonly DateTime _unixEpoch = new DateTime(1970, 1, 1); private readonly DateTimeTransformer _parent; private ConcurrentDictionary _cache; private ConcurrentQueue _oldestKeys; - private readonly DateTime _unixEpoc; -#endregion + #endregion public Mapper(DateTimeTransformer parent, DataViewSchema inputSchema) : base(parent.Host.Register(nameof(Mapper)), inputSchema, parent) { _parent = parent; - _unixEpoc = new DateTime(1970, 1, 1); _cache = new ConcurrentDictionary(); _oldestKeys = new ConcurrentQueue(); } @@ -683,82 +682,98 @@ protected override DataViewSchema.DetachedColumn[] GetOutputColumnsCore() private Delegate MakeGetter(DataViewRow input, int iinfo) { - var getter = input.GetGetter(input.Schema[_parent._column.Source]); + // If already in posix time. + if (typeof(TInput) == typeof(long)) + return MakeLongGetter(input, iinfo); + // System.DateTime + else + return MakeDateTimeGetter(input, iinfo); + } + private Delegate MakeLongGetter(DataViewRow input, int iinfo) + { + var getter = input.GetGetter(input.Schema[_parent._column.Source]); ValueGetter result = (ref TTransformed dst) => { - TInput dateTimeColumn = default; - getter(ref dateTimeColumn); + long dateTime = default; + getter(ref dateTime); - long dateTime; + var timePoint = _parent._column.Transform(dateTime); - // Type already in Posix format - if(typeof(TInput) == typeof(long)) - dateTime = (long)(object)dateTimeColumn; - // System.DateTime - else - dateTime = ((DateTime)(object)dateTimeColumn).Subtract(_unixEpoc).Ticks / TimeSpan.TicksPerSecond; + dst = GetColumnFromStruct(ref timePoint, iinfo); + }; - var timePoint = _parent._column.Transform(dateTime); + return result; + } + + private Delegate MakeDateTimeGetter(DataViewRow input, int iinfo) + { + var getter = input.GetGetter(input.Schema[_parent._column.Source]); + ValueGetter result = (ref TTransformed dst) => + { + DateTime dateTime = default; + getter(ref dateTime); - if (iinfo == 0) - dst = (TTransformed)Convert.ChangeType(timePoint.Year, typeof(TTransformed)); - else if (iinfo == 1) - dst = (TTransformed)Convert.ChangeType(timePoint.Month, typeof(TTransformed)); - else if (iinfo == 2) - dst = (TTransformed)Convert.ChangeType(timePoint.Day, typeof(TTransformed)); - else if (iinfo == 3) - dst = (TTransformed)Convert.ChangeType(timePoint.Hour, typeof(TTransformed)); - else if (iinfo == 4) - dst = (TTransformed)Convert.ChangeType(timePoint.Minute, typeof(TTransformed)); - else if (iinfo == 5) - dst = (TTransformed)Convert.ChangeType(timePoint.Second, typeof(TTransformed)); - else if (iinfo == 6) - dst = (TTransformed)Convert.ChangeType(timePoint.AmPm, typeof(TTransformed)); - else if (iinfo == 7) - dst = (TTransformed)Convert.ChangeType(timePoint.Hour12, typeof(TTransformed)); - else if (iinfo == 8) - dst = (TTransformed)Convert.ChangeType(timePoint.DayOfWeek, typeof(TTransformed)); - else if (iinfo == 9) - dst = (TTransformed)Convert.ChangeType(timePoint.DayOfQuarter, typeof(TTransformed)); - else if (iinfo == 10) - dst = (TTransformed)Convert.ChangeType(timePoint.DayOfYear, typeof(TTransformed)); - else if (iinfo == 11) - dst = (TTransformed)Convert.ChangeType(timePoint.WeekOfMonth, typeof(TTransformed)); - else if (iinfo == 12) - dst = (TTransformed)Convert.ChangeType(timePoint.QuarterOfYear, typeof(TTransformed)); - else if (iinfo == 13) - dst = (TTransformed)Convert.ChangeType(timePoint.HalfOfYear, typeof(TTransformed)); - else if (iinfo == 14) - dst = (TTransformed)Convert.ChangeType(timePoint.WeekIso, typeof(TTransformed)); - else if (iinfo == 15) - dst = (TTransformed)Convert.ChangeType(timePoint.YearIso, typeof(TTransformed)); - else if (iinfo == 16) - dst = (TTransformed)Convert.ChangeType(timePoint.MonthLabel.AsMemory(), typeof(TTransformed)); - else if (iinfo == 17) - dst = (TTransformed)Convert.ChangeType(timePoint.AmPmLabel.AsMemory(), typeof(TTransformed)); - else if (iinfo == 18) - dst = (TTransformed)Convert.ChangeType(timePoint.DayOfWeekLabel.AsMemory(), typeof(TTransformed)); - else if (iinfo == 19) - dst = (TTransformed)Convert.ChangeType(timePoint.HolidayName.AsMemory(), typeof(TTransformed)); - else - dst = (TTransformed)Convert.ChangeType(timePoint.IsPaidTimeOff, typeof(TTransformed)); + var timePoint = _parent._column.Transform(dateTime.Subtract(_unixEpoch).Ticks / TimeSpan.TicksPerSecond); + + dst = GetColumnFromStruct(ref timePoint, iinfo); }; return result; } + private TTransformed GetColumnFromStruct(ref TimePoint timePoint, int iinfo) + { + if (iinfo == 0) + return (TTransformed)Convert.ChangeType(timePoint.Year, typeof(TTransformed)); + else if (iinfo == 1) + return (TTransformed)Convert.ChangeType(timePoint.Month, typeof(TTransformed)); + else if (iinfo == 2) + return (TTransformed)Convert.ChangeType(timePoint.Day, typeof(TTransformed)); + else if (iinfo == 3) + return (TTransformed)Convert.ChangeType(timePoint.Hour, typeof(TTransformed)); + else if (iinfo == 4) + return (TTransformed)Convert.ChangeType(timePoint.Minute, typeof(TTransformed)); + else if (iinfo == 5) + return (TTransformed)Convert.ChangeType(timePoint.Second, typeof(TTransformed)); + else if (iinfo == 6) + return (TTransformed)Convert.ChangeType(timePoint.AmPm, typeof(TTransformed)); + else if (iinfo == 7) + return (TTransformed)Convert.ChangeType(timePoint.Hour12, typeof(TTransformed)); + else if (iinfo == 8) + return (TTransformed)Convert.ChangeType(timePoint.DayOfWeek, typeof(TTransformed)); + else if (iinfo == 9) + return (TTransformed)Convert.ChangeType(timePoint.DayOfQuarter, typeof(TTransformed)); + else if (iinfo == 10) + return (TTransformed)Convert.ChangeType(timePoint.DayOfYear, typeof(TTransformed)); + else if (iinfo == 11) + return (TTransformed)Convert.ChangeType(timePoint.WeekOfMonth, typeof(TTransformed)); + else if (iinfo == 12) + return (TTransformed)Convert.ChangeType(timePoint.QuarterOfYear, typeof(TTransformed)); + else if (iinfo == 13) + return (TTransformed)Convert.ChangeType(timePoint.HalfOfYear, typeof(TTransformed)); + else if (iinfo == 14) + return (TTransformed)Convert.ChangeType(timePoint.WeekIso, typeof(TTransformed)); + else if (iinfo == 15) + return (TTransformed)Convert.ChangeType(timePoint.YearIso, typeof(TTransformed)); + else if (iinfo == 16) + return (TTransformed)Convert.ChangeType(timePoint.MonthLabel.AsMemory(), typeof(TTransformed)); + else if (iinfo == 17) + return (TTransformed)Convert.ChangeType(timePoint.AmPmLabel.AsMemory(), typeof(TTransformed)); + else if (iinfo == 18) + return (TTransformed)Convert.ChangeType(timePoint.DayOfWeekLabel.AsMemory(), typeof(TTransformed)); + else if (iinfo == 19) + return (TTransformed)Convert.ChangeType(timePoint.HolidayName.AsMemory(), typeof(TTransformed)); + else + return (TTransformed)Convert.ChangeType(timePoint.IsPaidTimeOff, typeof(TTransformed)); + } + protected override Delegate MakeGetter(DataViewRow input, int iinfo, Func activeOutput, out Action disposer) { disposer = null; // Have to add 1 to iinfo since the enum starts at 1 - // If input type is long, Posix time - if (input.Schema[_parent._column.Source].Type.RawType == typeof(long)) - return Utils.MarshalInvoke(MakeGetter, new Type[] { typeof(long), ((DateTimeEstimator.ColumnsProduced)iinfo + 1).GetRawColumnType() }, input, iinfo); - // If input types is System.DateTime - else - return Utils.MarshalInvoke(MakeGetter, new Type[] { typeof(DateTime), ((DateTimeEstimator.ColumnsProduced)iinfo + 1).GetRawColumnType() }, input, iinfo); + return Utils.MarshalInvoke(MakeGetter, new Type[] { input.Schema[_parent._column.Source].Type.RawType, ((DateTimeEstimator.ColumnsProduced)iinfo + 1).GetRawColumnType() }, input, iinfo); } diff --git a/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs b/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs index d28b844799..a5190f987e 100644 --- a/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs +++ b/test/Microsoft.ML.Tests/Transformers/DateTimeTransformerTests.cs @@ -93,7 +93,7 @@ public void CorrectNumberOfColumnsAndSchema() [NotCentOS7Fact] public void CanUseDateFromColumnLongType() { - // Future Date - 2025 June 30 + // Date - 2025 June 30 MLContext mlContext = new MLContext(1); var dataList = new[] { new DateTimeInput() { date = 1751241600 } }; var data = mlContext.Data.LoadFromEnumerable(dataList); @@ -136,7 +136,7 @@ public void CanUseDateFromColumnLongType() [NotCentOS7Fact] public void CanUseDateFromColumnDateTimeType() { - // Future Date - 2025 June 30 + // Date - 2025 June 30 MLContext mlContext = new MLContext(1); var dataList = new[] { new { date = new DateTime(2025,6,30)} }; var data = mlContext.Data.LoadFromEnumerable(dataList); @@ -179,7 +179,7 @@ public void CanUseDateFromColumnDateTimeType() [NotCentOS7Fact] public void HolidayTest() { - // Future Date - 2025 June 30 + // Date - 2025 June 30 MLContext mlContext = new MLContext(1); var dataList = new[] { new DateTimeInput() { date = 157161600 } }; var data = mlContext.Data.LoadFromEnumerable(dataList); @@ -203,7 +203,7 @@ public void HolidayTest() [NotCentOS7Fact] public void ManyRowsTestLongType() { - // Future Date - 2025 June 30 + // Date - 2025 June 30 MLContext mlContext = new MLContext(1); var dataList = new[] { new DateTimeInput() { date = 1751241600 }, new DateTimeInput() { date = 1751241600 }, new DateTimeInput() { date = 12341 }, new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 134 }, new DateTimeInput() { date = 1234 }, new DateTimeInput() { date = 1751241600 }, @@ -250,7 +250,7 @@ public void ManyRowsTestLongType() [NotCentOS7Fact] public void ManyRowsTestDateTimeType() { - // Future Date - 2025 June 30 + // Date - 2025 June 30 MLContext mlContext = new MLContext(1); var dataList = new[] { new { date = new DateTime(2025, 6, 30) }, new { date = new DateTime(2025, 6, 30) }, new { date = DateTime.Now }, new { date = DateTime.UtcNow }, @@ -296,7 +296,7 @@ public void ManyRowsTestDateTimeType() [NotCentOS7Fact] public void EntryPointTest() { - // Future Date - 2025 June 30 + // Date - 2025 June 30 MLContext mlContext = new MLContext(1); var dataList = new[] { new DateTimeInput() { date = 1751241600 } }; var data = mlContext.Data.LoadFromEnumerable(dataList); From c6d7fe17bb153840e0952d6550454c9c5b5e907d Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Tue, 21 Jan 2020 10:25:46 -0800 Subject: [PATCH 3/3] added in new struct to mirror native struct --- .../DateTimeTransformer.cs | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs b/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs index d0748e3fce..016ecea37b 100644 --- a/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs +++ b/src/Microsoft.ML.Featurizers/DateTimeTransformer.cs @@ -354,6 +354,37 @@ protected override bool ReleaseHandle() #region TimePoint + // Exact native representation + [StructLayout(LayoutKind.Sequential, Pack = 1)] + internal struct NativeTimePoint + { + public int Year; + public byte Month; + public byte Day; + public byte Hour; + public byte Minute; + public byte Second; + public byte AmPm; + public byte Hour12; + public byte DayOfWeek; + public byte DayOfQuarter; + public ushort DayOfYear; + public ushort WeekOfMonth; + public byte QuarterOfYear; + public byte HalfOfYear; + public byte WeekIso; + public int YearIso; + public IntPtr MonthLabelPointer; + public IntPtr MonthLabelSize; + public IntPtr AmPmLabelPointer; + public IntPtr AmPmLabelSize; + public IntPtr DayOfWeekLabelPointer; + public IntPtr DayOfWeekLabelSize; + public IntPtr HolidayNamePointer; + public IntPtr HolidayNameSize; + public byte IsPaidTimeOff; + } + [StructLayoutAttribute(LayoutKind.Sequential)] internal struct TimePoint { @@ -468,14 +499,13 @@ internal abstract class TypedColumn : IDisposable internal readonly int IntPtrSize; internal readonly int StructSize; - internal TypedColumn(string source, string prefix) + internal unsafe TypedColumn(string source, string prefix) { Source = source; Prefix = prefix; IntPtrSize = IntPtr.Size; - // The native struct is 25 bytes + 8 size_t. - StructSize = 25 + (IntPtrSize * 8); + StructSize = sizeof(NativeTimePoint); } internal abstract void CreateTransformerFromEstimator(DateTimeEstimator.HolidayList country);