diff --git a/README.md b/README.md index 510c1d1..c73acf4 100644 --- a/README.md +++ b/README.md @@ -253,13 +253,14 @@ SQL three-valued logic; date columns compare against `'yyyy-MM-dd'` or a stable in-memory sort of the matching rows; `TOP`/`LIMIT` applies after the sort. When a sidecar compound index (`file.cdx`) exists next to the table, queries use it -automatically: equality, range, `BETWEEN` and prefix `LIKE` predicates on indexed -character columns become index seeks, and an `ORDER BY` matching an index tag reads in -index order instead of sorting. This applies to SQL text and to the `Query` builder -alike. The planner is conservative — index tags with dBASE `UNIQUE` or `FOR` filters, -descending or non-character keys, expression keys, or non-ASCII search values fall back -to a full table scan, and the full `WHERE` clause is always re-applied to every row an -index returns. Set `UseIndexes=false` in the connection string (or call +automatically: equality, range and `BETWEEN` predicates on indexed character, integer, +numeric, double and date columns (plus prefix `LIKE` on character columns) become index +seeks, and an `ORDER BY` matching an index tag reads in index order instead of sorting. +This applies to SQL text and to the `Query` builder alike. The planner is +conservative — index tags with dBASE `UNIQUE` or `FOR` filters, descending keys, +expression keys, unsupported key types (datetime, currency), or non-ASCII character +search values fall back to a full table scan, and the full `WHERE` clause is always +re-applied to every row an index returns. Set `UseIndexes=false` in the connection string (or call `.WithoutIndexes()` on the builder) to force scans, and use `DbfDbCommand.ExplainPlan()` or `DbfQuery.ExplainPlan()` to see which path a query takes: diff --git a/src/DbfDataReader/Cdx/CdxKeyComparer.cs b/src/DbfDataReader/Cdx/CdxKeyComparer.cs index db23a0c..18554f7 100644 --- a/src/DbfDataReader/Cdx/CdxKeyComparer.cs +++ b/src/DbfDataReader/Cdx/CdxKeyComparer.cs @@ -2,16 +2,22 @@ namespace DbfDataReader.Cdx { internal static class CdxKeyComparer { - private const byte Pad = 0x20; + private const byte CharacterPad = 0x20; // Compares a stored index key against a full-length target key. Stored leaf keys have // their trailing padding trimmed, so missing trailing bytes compare as the pad byte - // otherwise a stored key that is a strict prefix of the target would compare as equal. + // Character keys pad with spaces; binary keys (integer, double, date) pad with zeros. public static int Compare(byte[] storedKey, byte[] targetKey) + { + return Compare(storedKey, targetKey, CharacterPad); + } + + public static int Compare(byte[] storedKey, byte[] targetKey, byte pad) { for (var i = 0; i < targetKey.Length; i++) { - var storedByte = i < storedKey.Length ? storedKey[i] : Pad; + var storedByte = i < storedKey.Length ? storedKey[i] : pad; var cmp = storedByte.CompareTo(targetKey[i]); if (cmp != 0) return cmp; } diff --git a/src/DbfDataReader/Cdx/CdxKeyEncoder.cs b/src/DbfDataReader/Cdx/CdxKeyEncoder.cs new file mode 100644 index 0000000..2e0b5ac --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxKeyEncoder.cs @@ -0,0 +1,87 @@ +using System; +using System.Buffers.Binary; + +namespace DbfDataReader.Cdx +{ + // Encodes search values into the binary key formats Visual FoxPro uses for + // non-character index tags, where unsigned byte-wise comparison matches value + // order: integer keys are big-endian with the sign bit flipped, and double-based + // keys (numeric, float, double, date) are big-endian IEEE 754 doubles with the + // sign bit flipped for non-negatives and every bit flipped for negatives. The + // integer format is verified against the tags in test/fixtures/foxprodb. + internal static class CdxKeyEncoder + { + public const int IntegerKeyLength = 4; + public const int DoubleKeyLength = 8; + + // days from 0001-01-01 to the start of the Julian period, matching the + // DateTime interpretation in DbfValueDateTime + private const int JulianDayOfDayOne = 1721426; + + private static readonly DateTime DayOne = new DateTime(1, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + + public static byte[] EncodeInteger(int value) + { + var key = new byte[IntegerKeyLength]; + BinaryPrimitives.WriteInt32BigEndian(key, value); + key[0] ^= 0x80; + + return key; + } + + public static int DecodeInteger(ReadOnlySpan key) + { + Span bytes = stackalloc byte[IntegerKeyLength]; + key.CopyTo(bytes); + bytes[0] ^= 0x80; + + return BinaryPrimitives.ReadInt32BigEndian(bytes); + } + + public static byte[] EncodeDouble(double value) + { + var key = new byte[DoubleKeyLength]; + BinaryPrimitives.WriteInt64BigEndian(key, BitConverter.DoubleToInt64Bits(value)); + + if ((key[0] & 0x80) == 0) + { + key[0] ^= 0x80; + } + else + { + for (var i = 0; i < key.Length; i++) key[i] ^= 0xFF; + } + + return key; + } + + public static double DecodeDouble(ReadOnlySpan key) + { + Span bytes = stackalloc byte[DoubleKeyLength]; + key.CopyTo(bytes); + + if ((bytes[0] & 0x80) != 0) + { + bytes[0] ^= 0x80; + } + else + { + for (var i = 0; i < bytes.Length; i++) bytes[i] ^= 0xFF; + } + + return BitConverter.Int64BitsToDouble(BinaryPrimitives.ReadInt64BigEndian(bytes)); + } + + // date and datetime keys are Julian day numbers (with the time of day as the + // fractional part) in the double key format + public static double ToJulianDay(DateTime value) + { + return (value.Date - DayOne).Days + JulianDayOfDayOne + value.TimeOfDay.TotalDays; + } + + public static byte[] EncodeDate(DateTime value) + { + return EncodeDouble(ToJulianDay(value)); + } + } +} diff --git a/src/DbfDataReader/Query/QueryPlanner.cs b/src/DbfDataReader/Query/QueryPlanner.cs index 7be62ca..3488d9e 100644 --- a/src/DbfDataReader/Query/QueryPlanner.cs +++ b/src/DbfDataReader/Query/QueryPlanner.cs @@ -14,7 +14,8 @@ namespace DbfDataReader.Query // filter, so an index result only has to be a superset of the matching rows. internal static class QueryPlanner { - private const byte Pad = 0x20; + private const byte CharacterPad = 0x20; + private const byte BinaryPad = 0x00; private enum CandidateKind { @@ -32,8 +33,8 @@ public static QueryAccessPlan CreatePlan(SqlExpression where, var orderOrdinal = GetSingleAscendingOrderOrdinal(orderKeys); if (where == null && orderOrdinal < 0) return QueryAccessPlan.FullScan("no indexable clause"); - // index keys compare byte-wise; only single-byte encodings keep that order - // consistent with the evaluator's ordinal string comparison + // character keys compare byte-wise; only single-byte encodings keep that + // order consistent with the evaluator's ordinal string comparison if (table.CurrentEncoding == null || !table.CurrentEncoding.IsSingleByte) return QueryAccessPlan.FullScan("multi-byte encoding"); @@ -75,7 +76,7 @@ private static QueryAccessPlan CreatePlanCore(SqlExpression where, int orderOrdi if (where == null && orderOrdinal >= 0 && tags.TryGetValue(orderOrdinal, out var orderTag)) { var entries = orderTag.Index.EnumerateEntries().ToList(); - StabilizeDuplicateKeyRuns(entries, orderTag.Index.Header.KeyLength); + StabilizeDuplicateKeyRuns(entries, orderTag.Index.Header.KeyLength, orderTag.PadByte); var recordIndexes = ToRecordIndexes(entries, sortSatisfied: true); return QueryAccessPlan.Index(recordIndexes, true, $"index order scan on tag '{orderTag.Name}'"); } @@ -91,16 +92,33 @@ private static int GetSingleAscendingOrderOrdinal(IReadOnlyList<(int Ordinal, bo : -1; } - // A tag is usable when its key expression is a plain character column, ordered - // ascending, and carries none of the flags that make its entries a subset of - // the table: dBASE-style UNIQUE indexes only the first record per key, and FOR - // clauses filter rows. Flag 0x04 (named CustomIndex here) is NOT excluded: in - // Visual FoxPro files it marks primary/candidate keys, which reject duplicate - // values instead of hiding records and therefore cover every row. - private static Dictionary FindEligibleTags(CdxFile cdxFile, - IList columns) + private sealed class IndexTag { - var tags = new Dictionary(); + public IndexTag(string name, CdxIndex index, DbfColumnType columnType) + { + Name = name; + Index = index; + ColumnType = columnType; + } + + public string Name { get; } + public CdxIndex Index { get; } + public DbfColumnType ColumnType { get; } + + public byte PadByte => ColumnType == DbfColumnType.Character ? CharacterPad : BinaryPad; + } + + // A tag is usable when its key expression is a plain column of a supported + // type, ordered ascending, and carries none of the flags that make its entries + // a subset of the table: dBASE-style UNIQUE indexes only the first record per + // key, and FOR clauses filter rows. Flag 0x04 (named CustomIndex here) is NOT + // excluded: in Visual FoxPro files it marks primary/candidate keys, which + // reject duplicate values instead of hiding records and therefore cover every + // row. Character keys may be any length; binary keys must have the width their + // encoding produces. + private static Dictionary FindEligibleTags(CdxFile cdxFile, IList columns) + { + var tags = new Dictionary(); foreach (var tagName in cdxFile.TagNames) { @@ -112,23 +130,44 @@ private static int GetSingleAscendingOrderOrdinal(IReadOnlyList<(int Ordinal, bo if (!IsPlainIdentifier(header.KeyExpression)) continue; var ordinal = FindColumn(columns, header.KeyExpression); - if (ordinal < 0 || columns[ordinal].ColumnType != DbfColumnType.Character) continue; + if (ordinal < 0) continue; + if (!IsSupportedKeyColumn(columns[ordinal].ColumnType, header.KeyLength)) continue; - if (!tags.ContainsKey(ordinal)) tags.Add(ordinal, (tagName, index)); + if (!tags.ContainsKey(ordinal)) + tags.Add(ordinal, new IndexTag(tagName, index, columns[ordinal].ColumnType)); } return tags; } + private static bool IsSupportedKeyColumn(DbfColumnType columnType, int keyLength) + { + switch (columnType) + { + case DbfColumnType.Character: + return true; + case DbfColumnType.SignedLong: + return keyLength == CdxKeyEncoder.IntegerKeyLength; + case DbfColumnType.Number: + case DbfColumnType.Float: + case DbfColumnType.Double: + case DbfColumnType.Date: + return keyLength == CdxKeyEncoder.DoubleKeyLength; + default: + return false; + } + } + private sealed class Candidate { - public Candidate(CandidateKind kind, int ordinal, CdxIndex index, string description, + public Candidate(CandidateKind kind, int ordinal, CdxIndex index, string description, byte padByte, byte[] equalityKey, Func comparison) { Kind = kind; Ordinal = ordinal; Index = index; Description = description; + PadByte = padByte; EqualityKey = equalityKey; Comparison = comparison; } @@ -137,13 +176,20 @@ public Candidate(CandidateKind kind, int ordinal, CdxIndex index, string descrip public int Ordinal { get; } public CdxIndex Index { get; } public string Description { get; } - public byte[] EqualityKey { get; } // null for an impossible equality (over-long key) + public byte PadByte { get; } + public byte[] EqualityKey { get; } // character equality searches public Func Comparison { get; } + + // no key and no comparison: a provably empty result + public static Candidate Empty(CandidateKind kind, IndexTag tag, int ordinal, string description) + { + return new Candidate(kind, ordinal, tag.Index, description, tag.PadByte, null, null); + } } - private static Candidate FindBestCandidate(SqlExpression where, - Dictionary tags, Encoding encoding, - IReadOnlyDictionary namedParameters, IReadOnlyList positionalParameters) + private static Candidate FindBestCandidate(SqlExpression where, Dictionary tags, + Encoding encoding, IReadOnlyDictionary namedParameters, + IReadOnlyList positionalParameters) { Candidate best = null; @@ -171,9 +217,9 @@ private static IEnumerable FlattenConjuncts(SqlExpression express } } - private static Candidate TryCreateCandidate(SqlExpression conjunct, - Dictionary tags, Encoding encoding, - IReadOnlyDictionary namedParameters, IReadOnlyList positionalParameters) + private static Candidate TryCreateCandidate(SqlExpression conjunct, Dictionary tags, + Encoding encoding, IReadOnlyDictionary namedParameters, + IReadOnlyList positionalParameters) { switch (conjunct) { @@ -234,13 +280,33 @@ private static SqlBinaryOperator Flip(SqlBinaryOperator op) } private static Candidate TryCreateComparisonCandidate(SqlColumnExpression column, - SqlExpression valueExpression, SqlBinaryOperator op, - Dictionary tags, Encoding encoding, + SqlExpression valueExpression, SqlBinaryOperator op, Dictionary tags, Encoding encoding, IReadOnlyDictionary namedParameters, IReadOnlyList positionalParameters) { if (!tags.TryGetValue(column.Ordinal, out var tag)) return null; - if (!TryResolveSearchText(valueExpression, namedParameters, positionalParameters, out var text)) - return null; + if (!TryResolveValue(valueExpression, namedParameters, positionalParameters, out var value)) return null; + + switch (tag.ColumnType) + { + case DbfColumnType.Character: + return TryCreateTextComparisonCandidate(column, value, op, tag, encoding); + case DbfColumnType.SignedLong: + return TryCreateIntegerComparisonCandidate(column, value, op, tag); + case DbfColumnType.Number: + case DbfColumnType.Float: + case DbfColumnType.Double: + return TryCreateDoubleComparisonCandidate(column, value, op, tag); + case DbfColumnType.Date: + return TryCreateDateComparisonCandidate(column, value, op, tag); + default: + return null; + } + } + + private static Candidate TryCreateTextComparisonCandidate(SqlColumnExpression column, object value, + SqlBinaryOperator op, IndexTag tag, Encoding encoding) + { + if (!TryGetSearchText(value, out var text)) return null; var keyLength = tag.Index.Header.KeyLength; var fits = TryPadKey(text, keyLength, encoding, out var target); @@ -248,20 +314,23 @@ private static Candidate TryCreateComparisonCandidate(SqlColumnExpression column if (op == SqlBinaryOperator.Equal) { // an equality value that cannot fit in the key is provably empty - return new Candidate(CandidateKind.Equality, column.Ordinal, tag.Index, - $"index seek (=) on tag '{tag.Name}'", fits ? target : null, null); + return fits + ? new Candidate(CandidateKind.Equality, column.Ordinal, tag.Index, + $"index seek (=) on tag '{tag.Name}'", tag.PadByte, target, null) + : Candidate.Empty(CandidateKind.Equality, tag, column.Ordinal, + $"index seek (=) on tag '{tag.Name}'"); } if (!fits) return null; // over-long bound: leave it to the scan - var comparison = CreateRangeComparison(op, target); + var comparison = CreateTextRangeComparison(op, target); if (comparison == null) return null; return new Candidate(CandidateKind.Range, column.Ordinal, tag.Index, - $"index range scan on tag '{tag.Name}'", null, comparison); + $"index range scan on tag '{tag.Name}'", tag.PadByte, null, comparison); } - private static Func CreateRangeComparison(SqlBinaryOperator op, byte[] target) + private static Func CreateTextRangeComparison(SqlBinaryOperator op, byte[] target) { switch (op) { @@ -278,15 +347,132 @@ private static Func CreateRangeComparison(SqlBinaryOperator op, byt } } + private static Candidate TryCreateIntegerComparisonCandidate(SqlColumnExpression column, object value, + SqlBinaryOperator op, IndexTag tag) + { + if (!TryGetNumber(value, out var number)) return null; + + if (op == SqlBinaryOperator.Equal) + { + // non-integral or out-of-range values cannot equal any integer column value + if (decimal.Truncate(number) != number || number < int.MinValue || number > int.MaxValue) + return Candidate.Empty(CandidateKind.Equality, tag, column.Ordinal, + $"index seek (=) on tag '{tag.Name}'"); + + var target = CdxKeyEncoder.EncodeInteger((int)number); + return new Candidate(CandidateKind.Equality, column.Ordinal, tag.Index, + $"index seek (=) on tag '{tag.Name}'", tag.PadByte, null, + stored => CompareBinary(stored, target)); + } + + // convert the bound to the integer domain; strict operators become + // inclusive against the adjacent integer + var isLowerBound = op == SqlBinaryOperator.GreaterThanOrEqual || op == SqlBinaryOperator.GreaterThan; + var bound = AdjustIntegerBound(number, op); + + if (isLowerBound && bound > int.MaxValue) + return Candidate.Empty(CandidateKind.Range, tag, column.Ordinal, + $"index range scan on tag '{tag.Name}'"); + if (!isLowerBound && bound < int.MinValue) + return Candidate.Empty(CandidateKind.Range, tag, column.Ordinal, + $"index range scan on tag '{tag.Name}'"); + + // a bound beyond the other end matches every row; a scan serves that better + if (isLowerBound && bound < int.MinValue) return null; + if (!isLowerBound && bound > int.MaxValue) return null; + + var key = CdxKeyEncoder.EncodeInteger((int)bound); + var comparison = isLowerBound ? GreaterOrEqual(key) : LessOrEqual(key); + + return new Candidate(CandidateKind.Range, column.Ordinal, tag.Index, + $"index range scan on tag '{tag.Name}'", tag.PadByte, null, comparison); + } + + private static decimal AdjustIntegerBound(decimal number, SqlBinaryOperator op) + { + switch (op) + { + case SqlBinaryOperator.GreaterThanOrEqual: return Math.Ceiling(number); + case SqlBinaryOperator.GreaterThan: return Math.Floor(number) + 1; + case SqlBinaryOperator.LessThanOrEqual: return Math.Floor(number); + default: return Math.Ceiling(number) - 1; // LessThan + } + } + + private static Candidate TryCreateDoubleComparisonCandidate(SqlColumnExpression column, object value, + SqlBinaryOperator op, IndexTag tag) + { + if (!TryGetNumber(value, out var number)) return null; + + return CreateDoubleKeyCandidate(column, CdxKeyEncoder.EncodeDouble((double)number), op, tag); + } + + private static Candidate TryCreateDateComparisonCandidate(SqlColumnExpression column, object value, + SqlBinaryOperator op, IndexTag tag) + { + if (!TryGetDate(value, out var date)) return null; + + return CreateDoubleKeyCandidate(column, CdxKeyEncoder.EncodeDate(date), op, tag); + } + + private static Candidate CreateDoubleKeyCandidate(SqlColumnExpression column, byte[] target, + SqlBinaryOperator op, IndexTag tag) + { + if (op == SqlBinaryOperator.Equal) + { + return new Candidate(CandidateKind.Equality, column.Ordinal, tag.Index, + $"index seek (=) on tag '{tag.Name}'", tag.PadByte, null, + stored => CompareBinary(stored, target)); + } + + // strict bounds stay inclusive at the key level: converting decimals to + // doubles can collapse a strict boundary onto the bound itself, and the + // residual filter drops the boundary rows exactly + var isLowerBound = op == SqlBinaryOperator.GreaterThanOrEqual || op == SqlBinaryOperator.GreaterThan; + var comparison = isLowerBound ? GreaterOrEqual(target) : LessOrEqual(target); + + return new Candidate(CandidateKind.Range, column.Ordinal, tag.Index, + $"index range scan on tag '{tag.Name}'", tag.PadByte, null, comparison); + } + private static Candidate TryCreateBetweenCandidate(SqlColumnExpression column, SqlBetweenExpression between, - Dictionary tags, Encoding encoding, + Dictionary tags, Encoding encoding, IReadOnlyDictionary namedParameters, IReadOnlyList positionalParameters) { if (!tags.TryGetValue(column.Ordinal, out var tag)) return null; - if (!TryResolveSearchText(between.Low, namedParameters, positionalParameters, out var lowText)) - return null; - if (!TryResolveSearchText(between.High, namedParameters, positionalParameters, out var highText)) - return null; + if (!TryResolveValue(between.Low, namedParameters, positionalParameters, out var lowValue)) return null; + if (!TryResolveValue(between.High, namedParameters, positionalParameters, out var highValue)) return null; + + var description = $"index range scan (between) on tag '{tag.Name}'"; + + switch (tag.ColumnType) + { + case DbfColumnType.Character: + return TryCreateTextBetweenCandidate(column, lowValue, highValue, tag, encoding, description); + case DbfColumnType.SignedLong: + return TryCreateIntegerBetweenCandidate(column, lowValue, highValue, tag, description); + case DbfColumnType.Number: + case DbfColumnType.Float: + case DbfColumnType.Double: + return TryGetNumber(lowValue, out var lowNumber) && TryGetNumber(highValue, out var highNumber) + ? CreateBinaryBetweenCandidate(column, CdxKeyEncoder.EncodeDouble((double)lowNumber), + CdxKeyEncoder.EncodeDouble((double)highNumber), tag, description) + : null; + case DbfColumnType.Date: + return TryGetDate(lowValue, out var lowDate) && TryGetDate(highValue, out var highDate) + ? CreateBinaryBetweenCandidate(column, CdxKeyEncoder.EncodeDate(lowDate), + CdxKeyEncoder.EncodeDate(highDate), tag, description) + : null; + default: + return null; + } + } + + private static Candidate TryCreateTextBetweenCandidate(SqlColumnExpression column, object lowValue, + object highValue, IndexTag tag, Encoding encoding, string description) + { + if (!TryGetSearchText(lowValue, out var lowText)) return null; + if (!TryGetSearchText(highValue, out var highText)) return null; var keyLength = tag.Index.Header.KeyLength; if (!TryPadKey(lowText, keyLength, encoding, out var low)) return null; @@ -298,17 +484,49 @@ int Comparison(byte[] stored) return CdxKeyComparer.Compare(stored, high) > 0 ? 1 : 0; } - return new Candidate(CandidateKind.Range, column.Ordinal, tag.Index, - $"index range scan (between) on tag '{tag.Name}'", null, Comparison); + return new Candidate(CandidateKind.Range, column.Ordinal, tag.Index, description, tag.PadByte, null, + Comparison); + } + + private static Candidate TryCreateIntegerBetweenCandidate(SqlColumnExpression column, object lowValue, + object highValue, IndexTag tag, string description) + { + if (!TryGetNumber(lowValue, out var lowNumber)) return null; + if (!TryGetNumber(highValue, out var highNumber)) return null; + + var low = Math.Ceiling(lowNumber); + var high = Math.Floor(highNumber); + + if (low > high || low > int.MaxValue || high < int.MinValue) + return Candidate.Empty(CandidateKind.Range, tag, column.Ordinal, description); + + var lowKey = CdxKeyEncoder.EncodeInteger((int)Math.Max(low, int.MinValue)); + var highKey = CdxKeyEncoder.EncodeInteger((int)Math.Min(high, int.MaxValue)); + + return CreateBinaryBetweenCandidate(column, lowKey, highKey, tag, description); + } + + private static Candidate CreateBinaryBetweenCandidate(SqlColumnExpression column, byte[] low, byte[] high, + IndexTag tag, string description) + { + int Comparison(byte[] stored) + { + if (CompareBinary(stored, low) < 0) return -1; + return CompareBinary(stored, high) > 0 ? 1 : 0; + } + + return new Candidate(CandidateKind.Range, column.Ordinal, tag.Index, description, tag.PadByte, null, + Comparison); } private static Candidate TryCreateLikeCandidate(SqlColumnExpression column, SqlLikeExpression like, - Dictionary tags, Encoding encoding, + Dictionary tags, Encoding encoding, IReadOnlyDictionary namedParameters, IReadOnlyList positionalParameters) { if (!tags.TryGetValue(column.Ordinal, out var tag)) return null; - if (!TryResolveSearchText(like.Pattern, namedParameters, positionalParameters, out var pattern)) - return null; + if (tag.ColumnType != DbfColumnType.Character) return null; + if (!TryResolveValue(like.Pattern, namedParameters, positionalParameters, out var value)) return null; + if (!TryGetSearchText(value, out var pattern)) return null; var wildcardIndex = pattern.IndexOfAny(new[] { '%', '_' }); if (wildcardIndex <= 0) return null; // no usable prefix @@ -321,24 +539,35 @@ private static Candidate TryCreateLikeCandidate(SqlColumnExpression column, SqlL { // the column can never hold a value starting with a prefix longer than // the key; provably empty - return new Candidate(CandidateKind.LikePrefix, column.Ordinal, tag.Index, - $"index prefix scan on tag '{tag.Name}' (impossible prefix)", null, - null); + return Candidate.Empty(CandidateKind.LikePrefix, tag, column.Ordinal, + $"index prefix scan on tag '{tag.Name}' (impossible prefix)"); } return new Candidate(CandidateKind.LikePrefix, column.Ordinal, tag.Index, - $"index prefix scan (like) on tag '{tag.Name}'", null, + $"index prefix scan (like) on tag '{tag.Name}'", tag.PadByte, null, stored => CdxKeyComparer.Compare(stored, prefixBytes)); } - // the search text must be a string whose characters are printable ASCII; that - // keeps byte order and the evaluator's ordinal comparison sign-consistent - private static bool TryResolveSearchText(SqlExpression expression, + private static int CompareBinary(byte[] stored, byte[] target) + { + return CdxKeyComparer.Compare(stored, target, BinaryPad); + } + + private static Func GreaterOrEqual(byte[] target) + { + return stored => CompareBinary(stored, target) >= 0 ? 0 : -1; + } + + private static Func LessOrEqual(byte[] target) + { + return stored => CompareBinary(stored, target) <= 0 ? 0 : 1; + } + + private static bool TryResolveValue(SqlExpression expression, IReadOnlyDictionary namedParameters, IReadOnlyList positionalParameters, - out string text) + out object value) { - text = null; - object value; + value = null; switch (expression) { @@ -358,6 +587,15 @@ private static bool TryResolveSearchText(SqlExpression expression, } if (value is char character) value = character.ToString(); + return value != null; + } + + // character search text must be printable ASCII; that keeps byte order and the + // evaluator's ordinal comparison sign-consistent + private static bool TryGetSearchText(object value, out string text) + { + text = null; + if (!(value is string candidate)) return false; if (!candidate.All(c => c >= 0x20 && c <= 0x7E)) return false; @@ -365,6 +603,50 @@ private static bool TryResolveSearchText(SqlExpression expression, return true; } + private static bool TryGetNumber(object value, out decimal number) + { + number = 0; + + switch (value) + { + case byte _: + case sbyte _: + case short _: + case ushort _: + case int _: + case uint _: + case long _: + case ulong _: + case decimal _: + number = Convert.ToDecimal(value, System.Globalization.CultureInfo.InvariantCulture); + return true; + case float _: + case double _: + var floating = Convert.ToDouble(value, System.Globalization.CultureInfo.InvariantCulture); + if (double.IsNaN(floating) || floating < (double)decimal.MinValue || + floating > (double)decimal.MaxValue) return false; + number = (decimal)floating; + return true; + default: + return false; + } + } + + private static bool TryGetDate(object value, out DateTime date) + { + switch (value) + { + case DateTime dateTime: + date = dateTime; + return true; + case string text: + return SqlValueComparer.TryParseDateTime(text, out date); + default: + date = default; + return false; + } + } + // trims trailing spaces, which the dialect ignores, and pads the value to the // key length. Fails when the trimmed value cannot fit in a key. private static bool TryPadKey(string text, int keyLength, Encoding encoding, out byte[] key) @@ -382,7 +664,7 @@ private static bool TryPadKey(string text, int keyLength, Encoding encoding, out var padded = new byte[keyLength]; Array.Copy(bytes, padded, bytes.Length); - for (var i = bytes.Length; i < keyLength; i++) padded[i] = Pad; + for (var i = bytes.Length; i < keyLength; i++) padded[i] = CharacterPad; key = padded; return true; @@ -392,34 +674,31 @@ private static IReadOnlyList ExecuteSearch(Candidate candidate, bool sortSa { List entries; - if (candidate.Kind == CandidateKind.Equality) + if (candidate.Comparison != null) { - // an equality key that cannot fit in the index key is provably empty - entries = candidate.EqualityKey == null - ? new List() - : candidate.Index.Search(candidate.EqualityKey).ToList(); + entries = candidate.Index.Search(candidate.Comparison).ToList(); } - else if (candidate.Comparison == null) + else if (candidate.EqualityKey != null) { - entries = new List(); + entries = candidate.Index.Search(candidate.EqualityKey).ToList(); } else { - entries = candidate.Index.Search(candidate.Comparison).ToList(); + entries = new List(); } - StabilizeDuplicateKeyRuns(entries, candidate.Index.Header.KeyLength); + StabilizeDuplicateKeyRuns(entries, candidate.Index.Header.KeyLength, candidate.PadByte); return ToRecordIndexes(entries, sortSatisfied); } // entries with equal keys carry no defined order in the index; sorting each run // by record index makes index order identical to a stable scan-and-sort - private static void StabilizeDuplicateKeyRuns(List entries, int keyLength) + private static void StabilizeDuplicateKeyRuns(List entries, int keyLength, byte padByte) { var start = 0; for (var i = 1; i <= entries.Count; i++) { - if (i < entries.Count && KeysEqual(entries[start], entries[i], keyLength)) continue; + if (i < entries.Count && KeysEqual(entries[start], entries[i], keyLength, padByte)) continue; if (i - start > 1) entries.Sort(start, i - start, @@ -429,19 +708,19 @@ private static void StabilizeDuplicateKeyRuns(List entries, int key } } - private static bool KeysEqual(CdxKeyEntry x, CdxKeyEntry y, int keyLength) + private static bool KeysEqual(CdxKeyEntry x, CdxKeyEntry y, int keyLength, byte padByte) { - var target = TryPadKeyBytes(y.KeyBytes, keyLength); - return CdxKeyComparer.Compare(x.KeyBytes, target) == 0; + var target = PadKeyBytes(y.KeyBytes, keyLength, padByte); + return CdxKeyComparer.Compare(x.KeyBytes, target, padByte) == 0; } - private static byte[] TryPadKeyBytes(byte[] bytes, int keyLength) + private static byte[] PadKeyBytes(byte[] bytes, int keyLength, byte padByte) { if (bytes.Length >= keyLength) return bytes; var padded = new byte[keyLength]; Array.Copy(bytes, padded, bytes.Length); - for (var i = bytes.Length; i < keyLength; i++) padded[i] = Pad; + for (var i = bytes.Length; i < keyLength; i++) padded[i] = padByte; return padded; } diff --git a/src/DbfDataReader/Query/SqlValueComparer.cs b/src/DbfDataReader/Query/SqlValueComparer.cs index a43f06f..8fa756c 100644 --- a/src/DbfDataReader/Query/SqlValueComparer.cs +++ b/src/DbfDataReader/Query/SqlValueComparer.cs @@ -69,6 +69,12 @@ private static bool IsNumber(object value) } } + internal static bool TryParseDateTime(string text, out DateTime value) + { + return DateTime.TryParseExact(text, DateTimeFormats, CultureInfo.InvariantCulture, + DateTimeStyles.None, out value); + } + private static DateTime ToDateTime(object value, int position) { switch (value) @@ -76,9 +82,7 @@ private static DateTime ToDateTime(object value, int position) case DateTime dateTime: return dateTime; case string text: - if (DateTime.TryParseExact(text, DateTimeFormats, CultureInfo.InvariantCulture, - DateTimeStyles.None, out var parsed)) - return parsed; + if (TryParseDateTime(text, out var parsed)) return parsed; throw new InvalidOperationException( $"Cannot convert '{text}' to a date at position {position}; " + diff --git a/test/DbfDataReader.Tests/CdxKeyEncoderTests.cs b/test/DbfDataReader.Tests/CdxKeyEncoderTests.cs new file mode 100644 index 0000000..bdb0306 --- /dev/null +++ b/test/DbfDataReader.Tests/CdxKeyEncoderTests.cs @@ -0,0 +1,115 @@ +using System; +using System.Linq; +using DbfDataReader.Cdx; +using Shouldly; +using Xunit; + +namespace DbfDataReader.Tests; + +public class CdxKeyEncoderTests +{ + [Theory] + [InlineData(0, new byte[] { 0x80, 0x00, 0x00, 0x00 })] + [InlineData(1, new byte[] { 0x80, 0x00, 0x00, 0x01 })] + [InlineData(256, new byte[] { 0x80, 0x00, 0x01, 0x00 })] + [InlineData(-1, new byte[] { 0x7F, 0xFF, 0xFF, 0xFF })] + [InlineData(int.MaxValue, new byte[] { 0xFF, 0xFF, 0xFF, 0xFF })] + [InlineData(int.MinValue, new byte[] { 0x00, 0x00, 0x00, 0x00 })] + public void Should_encode_integers(int value, byte[] expected) + { + CdxKeyEncoder.EncodeInteger(value).ShouldBe(expected); + CdxKeyEncoder.DecodeInteger(expected).ShouldBe(value); + } + + [Fact] + public void Integer_key_byte_order_should_match_value_order() + { + var values = new[] { int.MinValue, -100000, -256, -1, 0, 1, 2, 255, 256, 100000, int.MaxValue }; + var keys = values.Select(CdxKeyEncoder.EncodeInteger).ToList(); + + for (var i = 1; i < keys.Count; i++) + { + CompareBytes(keys[i - 1], keys[i]).ShouldBeLessThan(0, $"{values[i - 1]} vs {values[i]}"); + } + } + + [Fact] + public void Should_encode_doubles_round_trip() + { + var values = new[] { double.MinValue, -1e10, -1.5, -1e-10, 0.0, 1e-10, 0.5, 1.0, 1e10, double.MaxValue }; + + foreach (var value in values) + { + var key = CdxKeyEncoder.EncodeDouble(value); + key.Length.ShouldBe(8); + CdxKeyEncoder.DecodeDouble(key).ShouldBe(value); + } + } + + [Fact] + public void Should_encode_known_doubles() + { + // 1.0 = 0x3FF0... big-endian, sign bit flipped + CdxKeyEncoder.EncodeDouble(1.0) + .ShouldBe(new byte[] { 0xBF, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }); + // -1.0 = 0xBFF0... fully complemented + CdxKeyEncoder.EncodeDouble(-1.0) + .ShouldBe(new byte[] { 0x40, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }); + CdxKeyEncoder.EncodeDouble(0.0) + .ShouldBe(new byte[] { 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }); + } + + [Fact] + public void Double_key_byte_order_should_match_value_order() + { + var values = new[] { -1e100, -2.0, -1.0, -0.5, -1e-10, 0.0, 1e-10, 0.5, 1.0, 2.0, 1e100 }; + var keys = values.Select(CdxKeyEncoder.EncodeDouble).ToList(); + + for (var i = 1; i < keys.Count; i++) + { + CompareBytes(keys[i - 1], keys[i]).ShouldBeLessThan(0, $"{values[i - 1]} vs {values[i]}"); + } + } + + [Theory] + [InlineData(1970, 1, 1, 2440588)] // well-known Julian day number anchors + [InlineData(2000, 1, 1, 2451545)] + [InlineData(1, 1, 1, 1721426)] + public void Should_convert_dates_to_julian_day_numbers(int year, int month, int day, int expectedJulianDay) + { + CdxKeyEncoder.ToJulianDay(new DateTime(year, month, day)).ShouldBe(expectedJulianDay); + } + + [Fact] + public void Should_include_the_time_of_day_as_a_fraction() + { + CdxKeyEncoder.ToJulianDay(new DateTime(2000, 1, 1, 12, 0, 0)).ShouldBe(2451545.5); + } + + [Fact] + public void Date_key_byte_order_should_match_date_order() + { + var dates = new[] + { + new DateTime(1899, 12, 30), new DateTime(1970, 1, 1), new DateTime(1997, 6, 15), + new DateTime(2000, 1, 1), new DateTime(2026, 7, 6) + }; + var keys = dates.Select(CdxKeyEncoder.EncodeDate).ToList(); + + for (var i = 1; i < keys.Count; i++) + { + CompareBytes(keys[i - 1], keys[i]).ShouldBeLessThan(0); + } + } + + private static int CompareBytes(byte[] x, byte[] y) + { + for (var i = 0; i < x.Length; i++) + { + var cmp = x[i].CompareTo(y[i]); + if (cmp != 0) return cmp; + } + + return 0; + } +} diff --git a/test/DbfDataReader.Tests/QueryIndexTests.cs b/test/DbfDataReader.Tests/QueryIndexTests.cs index 6c73c51..c43f58d 100644 --- a/test/DbfDataReader.Tests/QueryIndexTests.cs +++ b/test/DbfDataReader.Tests/QueryIndexTests.cs @@ -153,8 +153,7 @@ public void Should_sort_in_memory_when_the_index_cannot_satisfy_the_order() } [Theory] - [InlineData("select * from setup.dbf where VALUE = 1", "no matching index tag")] // integer-keyed tags unsupported - [InlineData("select * from calls.dbf where CONTACT_ID = 1", "no usable index tags")] + [InlineData("select * from setup.dbf where VALUE = 1", "no matching index tag")] // VALUE has no tag [InlineData("select * from setup.dbf where KEY_NAME <> 'CALLS'", "no matching index tag")] [InlineData("select * from setup.dbf where KEY_NAME like '%S'", "no matching index tag")] // no usable prefix public void Should_fall_back_to_a_scan_for_unindexable_predicates(string commandText, string expectedReason) diff --git a/test/DbfDataReader.Tests/QueryNumericIndexTests.cs b/test/DbfDataReader.Tests/QueryNumericIndexTests.cs new file mode 100644 index 0000000..470f439 --- /dev/null +++ b/test/DbfDataReader.Tests/QueryNumericIndexTests.cs @@ -0,0 +1,189 @@ +using System.Collections.Generic; +using System.Linq; +using DbfDataReader.Cdx; +using Shouldly; +using Xunit; + +namespace DbfDataReader.Tests; + +// Differential harness for integer-keyed index tags, against calls.dbf/calls.CDX: +// CALL_ID (unique values 1..16) and CONTACT_ID (duplicated foreign keys). Every query +// runs through the index path and a forced full scan and must return identical rows. +[Collection("foxprodb")] +public class QueryNumericIndexTests +{ + private const string FolderPath = "../../../../fixtures/foxprodb"; + + private static DbfDbConnection OpenConnection(bool useIndexes) + { + var connection = new DbfDbConnection(); + connection.ConnectionString = $"Folder={FolderPath};SkipDeletedRecords=false;UseIndexes={useIndexes}"; + connection.Open(); + return connection; + } + + private static List QueryRows(bool useIndexes, string commandText, + (string Name, object Value)? parameter = null) + { + using var connection = OpenConnection(useIndexes); + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + if (parameter != null) command.Parameters.AddWithValue(parameter.Value.Name, parameter.Value.Value); + + var rows = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + rows.Add(string.Join("|", values)); + } + + return rows; + } + + private static string Explain(string commandText, (string Name, object Value)? parameter = null) + { + using var connection = OpenConnection(useIndexes: true); + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + if (parameter != null) command.Parameters.AddWithValue(parameter.Value.Name, parameter.Value.Value); + + return command.ExplainPlan(); + } + + private static void ShouldMatchScan(string commandText, string expectedPlanFragment, int? expectedRows = null, + (string Name, object Value)? parameter = null) + { + Explain(commandText, parameter).ShouldContain(expectedPlanFragment); + + var indexed = QueryRows(useIndexes: true, commandText, parameter); + var scanned = QueryRows(useIndexes: false, commandText, parameter); + + indexed.ShouldBe(scanned); + if (expectedRows != null) indexed.Count.ShouldBe(expectedRows.Value); + } + + [Fact] + public void Fixture_keys_should_round_trip_through_the_integer_transform() + { + // decode every key in both integer tags and compare with the column value of + // the record it points at - direct proof the transform matches what VFP wrote + using var dbfTable = new DbfTable($"{FolderPath}/calls.dbf"); + var rows = new List<(long? CallId, long? ContactId)>(); + var record = new DbfRecord(dbfTable); + while (dbfTable.Read(record)) rows.Add(((long?)record.GetValue(0), (long?)record.GetValue(1))); + + using var cdxFile = new CdxFile($"{FolderPath}/calls.CDX", dbfTable.CurrentEncoding); + var verified = 0; + + foreach (var (tagName, select) in new (string, System.Func<(long? CallId, long? ContactId), long?>)[] + { + ("CALL_ID", row => row.CallId), + ("CONTACT_ID", row => row.ContactId) + }) + { + foreach (var entry in cdxFile.GetIndex(tagName).EnumerateEntries()) + { + entry.KeyBytes.Length.ShouldBe(4); + var decoded = CdxKeyEncoder.DecodeInteger(entry.KeyBytes); + decoded.ShouldBe((int)select(rows[entry.RecordIndex]).Value, $"tag {tagName}"); + verified++; + } + } + + verified.ShouldBe(32); // 16 records in each tag + } + + [Theory] + [InlineData("select CALL_ID, SUBJECT from calls.dbf where CALL_ID = 7", 1)] + [InlineData("select CALL_ID from calls.dbf where CONTACT_ID = 1", 5)] // duplicate key run + [InlineData("select CALL_ID from calls.dbf where CONTACT_ID = 99", 0)] + [InlineData("select CALL_ID from calls.dbf where CALL_ID = 1.5", 0)] // non-integral: provably empty + [InlineData("select CALL_ID from calls.dbf where CALL_ID = 5000000000", 0)] // beyond int range + public void Should_seek_integer_equality_through_the_index(string commandText, int expectedRows) + { + ShouldMatchScan(commandText, "index seek (=)", expectedRows); + } + + [Theory] + [InlineData("select CALL_ID from calls.dbf where CALL_ID >= 10")] + [InlineData("select CALL_ID from calls.dbf where CALL_ID > 10")] + [InlineData("select CALL_ID from calls.dbf where CALL_ID <= 5")] + [InlineData("select CALL_ID from calls.dbf where CALL_ID < 5")] + [InlineData("select CALL_ID from calls.dbf where CALL_ID >= 9.5")] // non-integral bound adjusts + [InlineData("select CALL_ID from calls.dbf where CALL_ID < 5.5")] + [InlineData("select CALL_ID from calls.dbf where 10 <= CALL_ID")] // flipped operands + public void Should_range_scan_integers_through_the_index(string commandText) + { + ShouldMatchScan(commandText, "index range scan"); + } + + [Fact] + public void Should_range_scan_integer_between_through_the_index() + { + ShouldMatchScan("select CALL_ID from calls.dbf where CALL_ID between 4 and 9", + "index range scan (between)", 6); + ShouldMatchScan("select CALL_ID from calls.dbf where CALL_ID between 8.5 and 8.9", + "index range scan (between)", 0); // no integer fits + } + + [Fact] + public void Should_seek_with_integer_parameters() + { + ShouldMatchScan("select CALL_ID from calls.dbf where CONTACT_ID = @id", + "index seek (=)", parameter: ("@id", 2)); + } + + [Fact] + public void Should_apply_residual_predicates_after_the_integer_index() + { + ShouldMatchScan("select CALL_ID from calls.dbf where CONTACT_ID = 1 and CALL_ID > 2", + "index seek (=)"); + } + + [Fact] + public void Should_satisfy_order_by_from_the_integer_index() + { + var commandText = "select CALL_ID from calls.dbf where CALL_ID >= 5 order by CALL_ID"; + + var plan = Explain(commandText); + plan.ShouldContain("index range scan"); + plan.ShouldNotContain("in-memory sort"); + + QueryRows(useIndexes: true, commandText).ShouldBe(QueryRows(useIndexes: false, commandText)); + } + + [Fact] + public void Should_order_by_a_duplicated_integer_key_stably() + { + // CONTACT_ID has duplicate runs; index order must equal the stable sort order + var commandText = "select CALL_ID, CONTACT_ID from calls.dbf order by CONTACT_ID"; + + Explain(commandText).ShouldContain("index order scan on tag 'CONTACT_ID'"); + + QueryRows(useIndexes: true, commandText).ShouldBe(QueryRows(useIndexes: false, commandText)); + } + + public class CallRow + { + public long? CALL_ID { get; set; } + public long? CONTACT_ID { get; set; } + } + + [Fact] + public void Should_use_the_integer_index_from_the_query_builder() + { + using var dbfTable = new DbfTable($"{FolderPath}/calls.dbf"); + + var query = dbfTable.Query().Where(c => c.CONTACT_ID == 1); + query.ExplainPlan().ShouldContain("index seek (=) on tag 'CONTACT_ID'"); + + var indexed = query.ToList(); + var scanned = dbfTable.Query().Where(c => c.CONTACT_ID == 1).WithoutIndexes().ToList(); + + indexed.Select(c => $"{c.CALL_ID}|{c.CONTACT_ID}") + .ShouldBe(scanned.Select(c => $"{c.CALL_ID}|{c.CONTACT_ID}")); + indexed.Count.ShouldBe(5); + } +} diff --git a/test/DbfDataReader.Tests/SqlParserTests.cs b/test/DbfDataReader.Tests/SqlParserTests.cs index fbcbfb6..0ed2024 100644 --- a/test/DbfDataReader.Tests/SqlParserTests.cs +++ b/test/DbfDataReader.Tests/SqlParserTests.cs @@ -14,9 +14,12 @@ public class SqlParserTests public SqlParserTests() { + // bare (unquoted) table names consist of word characters and dots; Bogus can + // generate names with apostrophes, ampersands or spaces, which would need + // quoting and are covered by the delimited-name tests var faker = new Faker(); var fileName = faker.System.FileName("dbf"); - _fileName = fileName.Replace("&", "_and_"); + _fileName = System.Text.RegularExpressions.Regex.Replace(fileName, @"[^\w.]", "_"); } [Fact]