diff --git a/README.md b/README.md index 94ef917..cd5ee47 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,39 @@ using (var dbfDataReader = new DbfDataReader(dbfPath)) } ``` +Visual FoxPro compound index files (`.cdx`) can be opened and searched, and search results +combined with `Seek` to jump straight to the matching records — `CdxKeyEntry.RecordIndex` +converts the index's one-based record numbers to the zero-based indexes `Seek` expects: + +```csharp +using DbfDataReader.Cdx; + +var dbfPath = "path/file.dbf"; +var cdxPath = "path/file.cdx"; + +using (var dbfTable = new DbfTable(dbfPath)) +using (var cdxFile = new CdxFile(cdxPath, dbfTable.CurrentEncoding)) +{ + var tagNames = cdxFile.TagNames; // the named indexes ("tags") in the file + + var index = cdxFile.GetIndex("CONTACT_ID"); // one tag; index.KeyExpression describes the key + var dbfRecord = new DbfRecord(dbfTable); + + foreach (var entry in index.Search("C0000000042")) + { + dbfTable.Seek(entry.RecordIndex); + dbfTable.Read(dbfRecord); + // dbfRecord now holds the matching row + } +} +``` + +`CdxIndex` also supports `EnumerateEntries()` (full in-order scan), `Count()`, and a +`Search(Func)` overload for range or prefix searches. Current limitations: +only ascending indexes with byte-wise (MACHINE collation) character keys are searchable, +index key expressions are exposed as text but not evaluated, and index entries include +deleted records (check `DbfRecord.IsDeleted` after seeking). + There is also an implementation of DbConnection so you can query a folder of files e.g. ```csharp diff --git a/src/DbfDataReader/Cdx/BaseCdxNode.cs b/src/DbfDataReader/Cdx/BaseCdxNode.cs new file mode 100644 index 0000000..14ff94c --- /dev/null +++ b/src/DbfDataReader/Cdx/BaseCdxNode.cs @@ -0,0 +1,56 @@ +using System; +using System.Buffers.Binary; +using System.Text; + +namespace DbfDataReader.Cdx +{ + internal abstract class BaseCdxNode + { + public const int NodeSize = 512; + public const int NoSibling = -1; + + protected BaseCdxNode(long offset, CdxIndexHeader indexHeader, CdxNodeAttributes attributes, int keyCount, + int leftSibling, int rightSibling) + { + Offset = offset; + IndexHeader = indexHeader; + Attributes = attributes; + KeyCount = keyCount; + LeftSibling = leftSibling; + RightSibling = rightSibling; + } + + public long Offset { get; } + + public CdxIndexHeader IndexHeader { get; } + + public CdxNodeAttributes Attributes { get; } + + public int KeyCount { get; } + + public int LeftSibling { get; } + + public int RightSibling { get; } + + public static BaseCdxNode Read(CdxIndexHeader indexHeader, long offset, ReadOnlySpan bytes, + Encoding encoding) + { + var attributes = (CdxNodeAttributes)BinaryPrimitives.ReadUInt16LittleEndian(bytes.Slice(0, 2)); + if ((attributes | CdxNodeAttributes.All) != CdxNodeAttributes.All) + throw new CdxException(CdxErrorCode.InvalidNodeAttributes); + + return attributes.HasFlag(CdxNodeAttributes.LeafNode) + ? LeafCdxNode.Read(indexHeader, offset, attributes, bytes, encoding) + : (BaseCdxNode)InteriorCdxNode.Read(indexHeader, offset, attributes, bytes); + } + + internal static (int KeyCount, int LeftSibling, int RightSibling) ReadCommonFields(ReadOnlySpan bytes) + { + var keyCount = BinaryPrimitives.ReadUInt16LittleEndian(bytes.Slice(2, 2)); + var leftSibling = BinaryPrimitives.ReadInt32LittleEndian(bytes.Slice(4, 4)); + var rightSibling = BinaryPrimitives.ReadInt32LittleEndian(bytes.Slice(8, 4)); + + return (keyCount, leftSibling, rightSibling); + } + } +} diff --git a/src/DbfDataReader/Cdx/CdxEnums.cs b/src/DbfDataReader/Cdx/CdxEnums.cs new file mode 100644 index 0000000..a85a3c2 --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxEnums.cs @@ -0,0 +1,40 @@ +using System; + +namespace DbfDataReader.Cdx +{ + [Flags] + public enum CdxIndexOptions : byte + { + None = 0, + Unique = 1, + // observed in real-world files (e.g. custom indexes), mentioned but not described in most CDX documentation + CustomIndex = 4, + HasForClause = 8, + BitVector = 16, + IsCompactIndex = 32, + IsCompoundIndexHeader = 64, + IsStructuralIndex = 128, + + All = Unique | CustomIndex | HasForClause | BitVector | IsCompactIndex | IsCompoundIndexHeader | + IsStructuralIndex + } + + public enum CdxIndexOrder + { + Ascending = 0, + Descending = 1 + } + + [Flags] + internal enum CdxNodeAttributes + { + // a node with neither RootNode nor LeafNode set is an interior node + None = 0, + RootNode = 1, + LeafNode = 2, + // observed in real-world files, undocumented + Unknown = 4, + + All = RootNode | LeafNode | Unknown + } +} diff --git a/src/DbfDataReader/Cdx/CdxException.cs b/src/DbfDataReader/Cdx/CdxException.cs new file mode 100644 index 0000000..0995d55 --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxException.cs @@ -0,0 +1,34 @@ +using System; + +namespace DbfDataReader.Cdx +{ + public class CdxException : Exception + { + public CdxException(CdxErrorCode code) + : base($"Invalid CDX index file: {code}") + { + Code = code; + } + + public CdxErrorCode Code { get; } + } + + public enum CdxErrorCode + { + None, + NotACompoundIndexHeader, + RootNodeDoesNotHaveRootAttribute, + LeftmostNodeHasLeftSibling, + InvalidNodeAttributes, + InvalidIndexOptions, + InvalidExpressionPoolLength, + InvalidInteriorNodeKeyCount, + InvalidInteriorNodeSibling, + InvalidLeafNodeKeyCount, + InvalidLeafNodeCalculatedKeyStartIndex, + FirstLeafNodeKeyEntryHasDuplicateBytes, + PackedKeyEntryLengthTooLong, + InteriorNodeHasNoKeyEntries, + NodeChainTooLong + } +} diff --git a/src/DbfDataReader/Cdx/CdxFile.cs b/src/DbfDataReader/Cdx/CdxFile.cs new file mode 100644 index 0000000..c6b7e8a --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxFile.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace DbfDataReader.Cdx +{ + public class CdxFile : Disposable + { + private readonly bool _leaveOpen; + private readonly long _startOffset; + private readonly byte[] _nodeBuffer = new byte[BaseCdxNode.NodeSize]; + + private Dictionary _taggedIndexes; + + public CdxFile(string path, Encoding encoding = null) + { + if (!File.Exists(path)) throw new FileNotFoundException(); + + Path = path; + CurrentEncoding = encoding ?? Encoding.ASCII; + Stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + + Init(); + } + + public CdxFile(Stream stream, Encoding encoding = null, bool leaveOpen = false) + { + Path = string.Empty; + CurrentEncoding = encoding ?? Encoding.ASCII; + Stream = stream; + _leaveOpen = leaveOpen; + _startOffset = stream.CanSeek ? stream.Position : 0; + + Init(); + } + + private void Init() + { + Header = ReadHeader(0); + if (!Header.Options.HasFlag(CdxIndexOptions.IsCompoundIndexHeader)) + throw new CdxException(CdxErrorCode.NotACompoundIndexHeader); + + RootNode = ReadNode(Header.RootNodePointer, Header); + } + + public string Path { get; } + + public Encoding CurrentEncoding { get; } + + public Stream Stream { get; private set; } + + public CdxIndexHeader Header { get; private set; } + + public bool IsClosed => Stream == null; + + // The file root node is a tag directory: its keys are tag names and its record + // numbers are the file offsets of the tag index headers. + internal BaseCdxNode RootNode { get; private set; } + + internal int MaxNodeCount => (int)(Stream.Length / BaseCdxNode.NodeSize) + 1; + + protected override void Dispose(bool disposing) + { + try + { + if (!disposing) return; + if (!_leaveOpen) Stream?.Dispose(); + } + finally + { + Stream = null; + } + } + + public IReadOnlyCollection TagNames + { + get + { + ReadTaggedIndexes(); + return _taggedIndexes.Keys; + } + } + + public IReadOnlyDictionary ReadTaggedIndexes() + { + if (_taggedIndexes != null) return _taggedIndexes; + + var tagDirectory = new CdxIndex(this, Header, RootNode); + + var taggedIndexes = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var entry in tagDirectory.EnumerateEntries()) + { + taggedIndexes[entry.Key] = ReadIndex(entry.RecordNumber); + } + + _taggedIndexes = taggedIndexes; + return _taggedIndexes; + } + + public CdxIndex GetIndex(string tagName) + { + if (tagName == null) throw new ArgumentNullException(nameof(tagName)); + + if (!ReadTaggedIndexes().TryGetValue(tagName, out var index)) + throw new ArgumentException($"Tag '{tagName}' was not found in the index file.", nameof(tagName)); + + return index; + } + + private CdxIndex ReadIndex(long offset) + { + var header = ReadHeader(offset); + if (!header.Options.HasFlag(CdxIndexOptions.IsCompoundIndexHeader)) + throw new CdxException(CdxErrorCode.NotACompoundIndexHeader); + + var rootNode = ReadNode(header.RootNodePointer, header); + if (!rootNode.Attributes.HasFlag(CdxNodeAttributes.RootNode)) + throw new CdxException(CdxErrorCode.RootNodeDoesNotHaveRootAttribute); + + return new CdxIndex(this, header, rootNode); + } + + private CdxIndexHeader ReadHeader(long offset) + { + var buffer = new byte[CdxIndexHeader.HeaderSize]; + Stream.Seek(_startOffset + offset, SeekOrigin.Begin); + Stream.ReadExactly(buffer, 0, buffer.Length); + + return new CdxIndexHeader(buffer, offset); + } + + internal BaseCdxNode ReadNode(long offset, CdxIndexHeader indexHeader) + { + Stream.Seek(_startOffset + offset, SeekOrigin.Begin); + Stream.ReadExactly(_nodeBuffer, 0, BaseCdxNode.NodeSize); + + return BaseCdxNode.Read(indexHeader, offset, _nodeBuffer, CurrentEncoding); + } + } +} diff --git a/src/DbfDataReader/Cdx/CdxIndex.cs b/src/DbfDataReader/Cdx/CdxIndex.cs new file mode 100644 index 0000000..9b9f59f --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxIndex.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace DbfDataReader.Cdx +{ + public class CdxIndex + { + private const byte Pad = 0x20; + + private readonly CdxFile _file; + private readonly BaseCdxNode _rootNode; + + internal CdxIndex(CdxFile file, CdxIndexHeader header, BaseCdxNode rootNode) + { + _file = file; + Header = header; + _rootNode = rootNode; + } + + public CdxIndexHeader Header { get; } + + public string KeyExpression => Header.KeyExpression; + + public string ForExpression => Header.ForExpression; + + public IEnumerable EnumerateEntries() + { + var leaf = GetLeftmostLeafNode(); + var chainLength = 0; + + while (true) + { + foreach (var entry in leaf.Entries) + { + yield return entry; + } + + if (leaf.RightSibling == BaseCdxNode.NoSibling) yield break; + + GuardNodeChain(ref chainLength); + leaf = (LeafCdxNode)ReadNode(leaf.RightSibling); + } + } + + public int Count() + { + var leaf = GetLeftmostLeafNode(); + var chainLength = 0; + var total = 0; + + while (true) + { + total += leaf.KeyCount; + + if (leaf.RightSibling == BaseCdxNode.NoSibling) return total; + + GuardNodeChain(ref chainLength); + leaf = (LeafCdxNode)ReadNode(leaf.RightSibling); + } + } + + public IEnumerable Search(string key) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + + return Search(PadKey(key)); + } + + public IEnumerable Search(byte[] key) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (key.Length != Header.KeyLength) + throw new ArgumentException( + $"Key length {key.Length} does not match the index key length {Header.KeyLength}.", nameof(key)); + + return Search(storedKey => CdxKeyComparer.Compare(storedKey, key)); + } + + // The comparison receives a stored key and returns less than zero if the stored key sorts + // before the wanted range, zero if it is within the range, and greater than zero if it + // sorts after the range. Stored leaf keys may be shorter than the index key length as + // their trailing padding is trimmed. + public IEnumerable Search(Func keyComparison) + { + if (keyComparison == null) throw new ArgumentNullException(nameof(keyComparison)); + if (Header.Order != CdxIndexOrder.Ascending) + throw new NotSupportedException("Searching descending indexes is not supported."); + + return SearchCore(keyComparison); + } + + private IEnumerable SearchCore(Func keyComparison) + { + var leaf = FindFirstCandidateLeaf(keyComparison); + if (leaf == null) yield break; + + // Scan leaf entries, following right siblings while matches can continue across nodes. + var chainLength = 0; + + while (true) + { + var entries = leaf.Entries; + var lastMatchedIndex = -1; + + for (var i = 0; i < entries.Count; i++) + { + var cmp = keyComparison(entries[i].KeyBytes); + if (cmp < 0) continue; + if (cmp > 0) yield break; + + yield return entries[i]; + lastMatchedIndex = i; + } + + if (lastMatchedIndex != entries.Count - 1) yield break; + if (leaf.RightSibling == BaseCdxNode.NoSibling) yield break; + + GuardNodeChain(ref chainLength); + leaf = (LeafCdxNode)ReadNode(leaf.RightSibling); + } + } + + // Interior entry keys are the upper bound of their subtree, so descend into the first + // entry that is not below the wanted range. Returns null when every entry is below it. + private LeafCdxNode FindFirstCandidateLeaf(Func keyComparison) + { + var node = _rootNode; + var depth = 0; + + while (node is InteriorCdxNode interiorNode) + { + if (interiorNode.KeyEntries.Count == 0) + throw new CdxException(CdxErrorCode.InteriorNodeHasNoKeyEntries); + + var entry = interiorNode.KeyEntries.FirstOrDefault(e => keyComparison(e.KeyBytes) >= 0); + if (entry == null) return null; + + GuardNodeChain(ref depth); + node = ReadNode(entry.NodePointer); + } + + return (LeafCdxNode)node; + } + + private LeafCdxNode GetLeftmostLeafNode() + { + var node = _rootNode; + var depth = 0; + + while (node is InteriorCdxNode interiorNode) + { + if (interiorNode.KeyEntries.Count == 0) + throw new CdxException(CdxErrorCode.InteriorNodeHasNoKeyEntries); + + GuardNodeChain(ref depth); + node = ReadNode(interiorNode.KeyEntries[0].NodePointer); + } + + var leaf = (LeafCdxNode)node; + if (leaf.LeftSibling != BaseCdxNode.NoSibling) + throw new CdxException(CdxErrorCode.LeftmostNodeHasLeftSibling); + + return leaf; + } + + private BaseCdxNode ReadNode(long offset) + { + return _file.ReadNode(offset, Header); + } + + private void GuardNodeChain(ref int chainLength) + { + chainLength++; + if (chainLength > _file.MaxNodeCount) throw new CdxException(CdxErrorCode.NodeChainTooLong); + } + + private byte[] PadKey(string key) + { + var bytes = _file.CurrentEncoding.GetBytes(key); + if (bytes.Length > Header.KeyLength) + throw new ArgumentException( + $"Key length {bytes.Length} exceeds the index key length {Header.KeyLength}.", nameof(key)); + if (bytes.Length == Header.KeyLength) return bytes; + + var padded = new byte[Header.KeyLength]; + Array.Copy(bytes, padded, bytes.Length); + for (var i = bytes.Length; i < padded.Length; i++) + { + padded[i] = Pad; + } + + return padded; + } + } +} diff --git a/src/DbfDataReader/Cdx/CdxIndexHeader.cs b/src/DbfDataReader/Cdx/CdxIndexHeader.cs new file mode 100644 index 0000000..505cc00 --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxIndexHeader.cs @@ -0,0 +1,68 @@ +using System; +using System.Buffers.Binary; +using System.Text; + +namespace DbfDataReader.Cdx +{ + public class CdxIndexHeader + { + public const int HeaderSize = 1024; + + internal CdxIndexHeader(ReadOnlySpan bytes, long offset) + { + Offset = offset; + + RootNodePointer = BinaryPrimitives.ReadUInt32LittleEndian(bytes.Slice(0, 4)); + FreeNodeListPointer = BinaryPrimitives.ReadInt32LittleEndian(bytes.Slice(4, 4)); + KeyLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.Slice(12, 2)); + Options = (CdxIndexOptions)bytes[14]; + Signature = bytes[15]; + + // 16 - 501 - reserved + Order = (CdxIndexOrder)BinaryPrimitives.ReadUInt16LittleEndian(bytes.Slice(502, 2)); + // 504 - 505 - reserved + var forExpressionLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.Slice(506, 2)); + // 508 - 509 - reserved + var keyExpressionLength = BinaryPrimitives.ReadUInt16LittleEndian(bytes.Slice(510, 2)); + + if ((Options | CdxIndexOptions.All) != CdxIndexOptions.All) + throw new CdxException(CdxErrorCode.InvalidIndexOptions); + + // the last 512 bytes are a pool holding the key expression followed by the FOR expression + var pool = bytes.Slice(512, 512); + if (keyExpressionLength + forExpressionLength > pool.Length) + throw new CdxException(CdxErrorCode.InvalidExpressionPoolLength); + + KeyExpression = ReadExpression(pool.Slice(0, keyExpressionLength)); + ForExpression = Options.HasFlag(CdxIndexOptions.HasForClause) + ? ReadExpression(pool.Slice(keyExpressionLength, forExpressionLength)) + : string.Empty; + } + + internal long Offset { get; } + + public long RootNodePointer { get; } + + public int FreeNodeListPointer { get; } + + public int KeyLength { get; } + + public CdxIndexOptions Options { get; } + + public byte Signature { get; } + + public CdxIndexOrder Order { get; } + + public string KeyExpression { get; } + + public string ForExpression { get; } + + private static string ReadExpression(ReadOnlySpan bytes) + { + var length = bytes.Length; + while (length > 0 && bytes[length - 1] == 0x00) length--; + + return Encoding.ASCII.GetString(bytes.Slice(0, length)); + } + } +} diff --git a/src/DbfDataReader/Cdx/CdxKeyComparer.cs b/src/DbfDataReader/Cdx/CdxKeyComparer.cs new file mode 100644 index 0000000..db23a0c --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxKeyComparer.cs @@ -0,0 +1,22 @@ +namespace DbfDataReader.Cdx +{ + internal static class CdxKeyComparer + { + private const byte Pad = 0x20; + + // Compares a stored index key against a full-length target key. Stored leaf keys have + // their trailing padding trimmed, so missing trailing bytes compare as the pad byte - + // otherwise a stored key that is a strict prefix of the target would compare as equal. + public static int Compare(byte[] storedKey, byte[] targetKey) + { + for (var i = 0; i < targetKey.Length; i++) + { + var storedByte = i < storedKey.Length ? storedKey[i] : Pad; + var cmp = storedByte.CompareTo(targetKey[i]); + if (cmp != 0) return cmp; + } + + return 0; + } + } +} diff --git a/src/DbfDataReader/Cdx/CdxKeyEntry.cs b/src/DbfDataReader/Cdx/CdxKeyEntry.cs new file mode 100644 index 0000000..5ceeabe --- /dev/null +++ b/src/DbfDataReader/Cdx/CdxKeyEntry.cs @@ -0,0 +1,33 @@ +using System.Text; + +namespace DbfDataReader.Cdx +{ + public class CdxKeyEntry + { + private readonly Encoding _encoding; + private string _key; + + internal CdxKeyEntry(byte[] keyBytes, int recordNumber, Encoding encoding) + { + KeyBytes = keyBytes; + RecordNumber = recordNumber; + _encoding = encoding; + } + + // the key value with its trailing index padding removed + public byte[] KeyBytes { get; } + + // one-based DBF record number, as stored in the index + public int RecordNumber { get; } + + // zero-based record index for use with DbfTable.Seek and DbfDataReader.Seek + public int RecordIndex => RecordNumber - 1; + + public string Key => _key ??= _encoding.GetString(KeyBytes, 0, KeyBytes.Length); + + public override string ToString() + { + return Key; + } + } +} diff --git a/src/DbfDataReader/Cdx/InteriorCdxNode.cs b/src/DbfDataReader/Cdx/InteriorCdxNode.cs new file mode 100644 index 0000000..8270c63 --- /dev/null +++ b/src/DbfDataReader/Cdx/InteriorCdxNode.cs @@ -0,0 +1,69 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; + +namespace DbfDataReader.Cdx +{ + internal sealed class InteriorCdxNode : BaseCdxNode + { + private const int KeyAreaOffset = 12; + private const int KeyAreaLength = 500; + + private InteriorCdxNode(long offset, CdxIndexHeader indexHeader, CdxNodeAttributes attributes, int keyCount, + int leftSibling, int rightSibling, InteriorCdxKeyEntry[] keyEntries) + : base(offset, indexHeader, attributes, keyCount, leftSibling, rightSibling) + { + KeyEntries = keyEntries; + } + + public IReadOnlyList KeyEntries { get; } + + public static InteriorCdxNode Read(CdxIndexHeader indexHeader, long offset, CdxNodeAttributes attributes, + ReadOnlySpan bytes) + { + var (keyCount, leftSibling, rightSibling) = ReadCommonFields(bytes); + + if (leftSibling < NoSibling || rightSibling < NoSibling) + throw new CdxException(CdxErrorCode.InvalidInteriorNodeSibling); + + // each entry is the key followed by two big-endian UInt32 values: the record number + // and the file offset of the child node (the documented "4 hex characters" IDX format + // does not apply to compound index interior nodes) + var entrySize = indexHeader.KeyLength + 8; + if (keyCount * entrySize > KeyAreaLength) + throw new CdxException(CdxErrorCode.InvalidInteriorNodeKeyCount); + + var keyArea = bytes.Slice(KeyAreaOffset, KeyAreaLength); + + var entries = new InteriorCdxKeyEntry[keyCount]; + for (var i = 0; i < keyCount; i++) + { + var entry = keyArea.Slice(i * entrySize, entrySize); + + var keyBytes = entry.Slice(0, indexHeader.KeyLength).ToArray(); + var recordNumber = BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(indexHeader.KeyLength, 4)); + var nodePointer = (int)BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(indexHeader.KeyLength + 4, 4)); + + entries[i] = new InteriorCdxKeyEntry(keyBytes, recordNumber, nodePointer); + } + + return new InteriorCdxNode(offset, indexHeader, attributes, keyCount, leftSibling, rightSibling, entries); + } + } + + internal sealed class InteriorCdxKeyEntry + { + public InteriorCdxKeyEntry(byte[] keyBytes, uint recordNumber, int nodePointer) + { + KeyBytes = keyBytes; + RecordNumber = recordNumber; + NodePointer = nodePointer; + } + + public byte[] KeyBytes { get; } + + public uint RecordNumber { get; } + + public int NodePointer { get; } + } +} diff --git a/src/DbfDataReader/Cdx/LeafCdxNode.cs b/src/DbfDataReader/Cdx/LeafCdxNode.cs new file mode 100644 index 0000000..3895396 --- /dev/null +++ b/src/DbfDataReader/Cdx/LeafCdxNode.cs @@ -0,0 +1,119 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Text; + +namespace DbfDataReader.Cdx +{ + internal sealed class LeafCdxNode : BaseCdxNode + { + private const int PackedAreaOffset = 24; + private const int PackedAreaLength = 488; + + private LeafCdxNode(long offset, CdxIndexHeader indexHeader, CdxNodeAttributes attributes, int keyCount, + int leftSibling, int rightSibling, CdxKeyEntry[] entries) + : base(offset, indexHeader, attributes, keyCount, leftSibling, rightSibling) + { + Entries = entries; + } + + public IReadOnlyList Entries { get; } + + public static LeafCdxNode Read(CdxIndexHeader indexHeader, long offset, CdxNodeAttributes attributes, + ReadOnlySpan bytes, Encoding encoding) + { + var (keyCount, leftSibling, rightSibling) = ReadCommonFields(bytes); + + var recordNumberMask = BinaryPrimitives.ReadUInt32LittleEndian(bytes.Slice(14, 4)); + var duplicateCountMask = bytes[18]; + var trailingCountMask = bytes[19]; + var recordNumberBits = bytes[20]; + var duplicateCountBits = bytes[21]; + var packedEntryLength = bytes[23]; + + if (packedEntryLength > 8) + throw new CdxException(CdxErrorCode.PackedKeyEntryLengthTooLong); + if (keyCount * packedEntryLength > PackedAreaLength) + throw new CdxException(CdxErrorCode.InvalidLeafNodeKeyCount); + + var packed = bytes.Slice(PackedAreaOffset, PackedAreaLength); + + // Packed entries sit at the front of the packed area; the new bytes of each key sit + // at the back, growing backwards. Keys are prefix-compressed against the previous key + // (duplicate count) and suffix-compressed by dropping trailing padding (trailing count). + var keyLength = indexHeader.KeyLength; + var keyValueSource = PackedAreaLength; + var entries = new CdxKeyEntry[keyCount]; + byte[] previousKey = null; + + for (var i = 0; i < keyCount; i++) + { + var packedEntry = CdxKeyPacking.ReadPackedEntry(packed, i * packedEntryLength, packedEntryLength); + + var recordNumber = (int)(packedEntry & recordNumberMask); + packedEntry >>= recordNumberBits; + var duplicateBytes = (int)(packedEntry & duplicateCountMask); + packedEntry >>= duplicateCountBits; + var trailingBytes = (int)(packedEntry & trailingCountMask); + + var newBytesCount = keyLength - duplicateBytes - trailingBytes; + keyValueSource -= newBytesCount; + + if (newBytesCount < 0 || keyValueSource < 0) + throw new CdxException(CdxErrorCode.InvalidLeafNodeCalculatedKeyStartIndex); + + var actualKeyLength = keyLength - trailingBytes; + var keyBytes = BuildKeyBytes(packed, keyValueSource, newBytesCount, duplicateBytes, actualKeyLength, + previousKey); + + entries[i] = new CdxKeyEntry(keyBytes, recordNumber, encoding); + previousKey = keyBytes; + } + + return new LeafCdxNode(offset, indexHeader, attributes, keyCount, leftSibling, rightSibling, entries); + } + + // A key is the first duplicateBytes of the previous key, then newBytesCount bytes taken + // from the back of the packed area, truncated to the key length minus trailing padding. + private static byte[] BuildKeyBytes(ReadOnlySpan packed, int keyValueSource, int newBytesCount, + int duplicateBytes, int actualKeyLength, byte[] previousKey) + { + var keyBytes = new byte[actualKeyLength]; + + if (duplicateBytes > 0) + { + if (previousKey == null) + throw new CdxException(CdxErrorCode.FirstLeafNodeKeyEntryHasDuplicateBytes); + + var duplicated = Math.Min(duplicateBytes, actualKeyLength); + for (var d = 0; d < duplicated; d++) + { + keyBytes[d] = previousKey[d]; + } + } + + for (int b = duplicateBytes, source = 0; source < newBytesCount && b < actualKeyLength; b++, source++) + { + keyBytes[b] = packed[keyValueSource + source]; + } + + return keyBytes; + } + } + + internal static class CdxKeyPacking + { + // A packed entry is a little-endian integer of up to eight bytes laid out, from the least + // significant bit, as: [record number][duplicate count][trailing count]. + public static long ReadPackedEntry(ReadOnlySpan bytes, int startIndex, int length) + { + long packedEntry = 0; + for (var i = length - 1; i >= 0; i--) + { + packedEntry = (packedEntry << 8) | bytes[startIndex + i]; + } + + return packedEntry; + } + } +} diff --git a/test/DbfDataReader.Tests/CdxKeyComparerTests.cs b/test/DbfDataReader.Tests/CdxKeyComparerTests.cs new file mode 100644 index 0000000..8beb41f --- /dev/null +++ b/test/DbfDataReader.Tests/CdxKeyComparerTests.cs @@ -0,0 +1,46 @@ +using System.Text; +using DbfDataReader.Cdx; +using Shouldly; +using Xunit; + +namespace DbfDataReader.Tests +{ + public class CdxKeyComparerTests + { + [Theory] + [InlineData("ABC", "ABC", 0)] + [InlineData("AB ", "AB ", 0)] + [InlineData("ABC", "ABD", -1)] + [InlineData("ABD", "ABC", 1)] + [InlineData("ABC", "AB ", 1)] + public void Should_compare_full_length_keys(string stored, string target, int expectedSign) + { + var cmp = CdxKeyComparer.Compare(Encoding.ASCII.GetBytes(stored), Encoding.ASCII.GetBytes(target)); + + if (expectedSign == 0) + cmp.ShouldBe(0); + else if (expectedSign < 0) + cmp.ShouldBeLessThan(0); + else + cmp.ShouldBeGreaterThan(0); + } + + [Fact] + public void Should_treat_trimmed_trailing_bytes_as_padding() + { + // stored leaf keys are trimmed of trailing spaces; "AB" is the stored form of "AB " + var stored = Encoding.ASCII.GetBytes("AB"); + + CdxKeyComparer.Compare(stored, Encoding.ASCII.GetBytes("AB ")).ShouldBe(0); + } + + [Fact] + public void Should_not_match_a_stored_key_that_is_a_prefix_of_the_target() + { + // "AB" (stored form of "AB ") sorts before "ABC" and must not compare as equal to it + var stored = Encoding.ASCII.GetBytes("AB"); + + CdxKeyComparer.Compare(stored, Encoding.ASCII.GetBytes("ABC")).ShouldBeLessThan(0); + } + } +} diff --git a/test/DbfDataReader.Tests/CdxKeyPackingTests.cs b/test/DbfDataReader.Tests/CdxKeyPackingTests.cs new file mode 100644 index 0000000..d1da34b --- /dev/null +++ b/test/DbfDataReader.Tests/CdxKeyPackingTests.cs @@ -0,0 +1,37 @@ +using DbfDataReader.Cdx; +using Shouldly; +using Xunit; + +namespace DbfDataReader.Tests +{ + public class CdxKeyPackingTests + { + [Theory] + [InlineData(0, 0, 0x00_00_00_00_00_00_00_00L)] + [InlineData(0, 1, 0x00_00_00_00_00_00_00_00L)] + [InlineData(0, 2, 0x00_00_00_00_00_00_01_00L)] + [InlineData(0, 3, 0x00_00_00_00_00_02_01_00L)] + [InlineData(0, 4, 0x00_00_00_00_03_02_01_00L)] + [InlineData(0, 5, 0x00_00_00_04_03_02_01_00L)] + [InlineData(0, 6, 0x00_00_05_04_03_02_01_00L)] + [InlineData(0, 7, 0x00_06_05_04_03_02_01_00L)] + [InlineData(0, 8, 0x07_06_05_04_03_02_01_00L)] + [InlineData(1, 8, 0x08_07_06_05_04_03_02_01L)] + [InlineData(2, 8, 0x09_08_07_06_05_04_03_02L)] + [InlineData(3, 8, 0x0A_09_08_07_06_05_04_03L)] + [InlineData(3, 7, 0x00_09_08_07_06_05_04_03L)] + [InlineData(3, 6, 0x00_00_08_07_06_05_04_03L)] + [InlineData(3, 5, 0x00_00_00_07_06_05_04_03L)] + [InlineData(3, 4, 0x00_00_00_00_06_05_04_03L)] + public void Should_read_packed_entries_correctly(int startIndex, int length, long expected) + { + var buffer = new byte[488]; + for (var i = 0; i < buffer.Length; i++) + { + buffer[i] = (byte)(i % 256); + } + + CdxKeyPacking.ReadPackedEntry(buffer, startIndex, length).ShouldBe(expected); + } + } +} diff --git a/test/DbfDataReader.Tests/CdxTests.cs b/test/DbfDataReader.Tests/CdxTests.cs new file mode 100644 index 0000000..bb99fd1 --- /dev/null +++ b/test/DbfDataReader.Tests/CdxTests.cs @@ -0,0 +1,219 @@ +using System; +using System.IO; +using System.Linq; +using DbfDataReader.Cdx; +using Shouldly; +using Xunit; + +namespace DbfDataReader.Tests +{ + [Collection("foxprodb")] + public class CdxTests + { + private const string FixturesPath = "../../../../fixtures/foxprodb"; + + public static TheoryData CdxFixtures => new TheoryData + { + "calls.CDX", + "contacts.CDX", + "setup.CDX", + "types.CDX", + "FOXPRO-DB-TEST.DCX" + }; + + [Theory] + [MemberData(nameof(CdxFixtures))] + public void Should_enumerate_tags(string fixtureName) + { + using var cdxFile = OpenCdx(fixtureName); + + cdxFile.TagNames.ShouldNotBeEmpty(); + + foreach (var tagName in cdxFile.TagNames) + { + var index = cdxFile.GetIndex(tagName); + + index.ShouldNotBeNull(); + index.Header.KeyLength.ShouldBeGreaterThan(0); + index.KeyExpression.ShouldNotBeNullOrEmpty(); + } + } + + [Theory] + [MemberData(nameof(CdxFixtures))] + public void Should_enumerate_entries_in_ascending_key_order(string fixtureName) + { + using var cdxFile = OpenCdx(fixtureName); + + foreach (var tagName in cdxFile.TagNames) + { + var index = cdxFile.GetIndex(tagName); + if (index.Header.Order != CdxIndexOrder.Ascending) continue; + + var entries = index.EnumerateEntries().ToList(); + entries.Count.ShouldBe(index.Count(), $"tag: {tagName}"); + + for (var i = 1; i < entries.Count; i++) + { + ComparePadded(entries[i - 1].KeyBytes, entries[i].KeyBytes, index.Header.KeyLength) + .ShouldBeLessThanOrEqualTo(0, $"tag: {tagName}, entry: {i}"); + } + } + } + + [Theory] + [MemberData(nameof(CdxFixtures))] + public void Should_find_entries_by_exact_key(string fixtureName) + { + using var cdxFile = OpenCdx(fixtureName); + var searches = 0; + + foreach (var tagName in cdxFile.TagNames) + { + var index = cdxFile.GetIndex(tagName); + if (index.Header.Order != CdxIndexOrder.Ascending) continue; + + var entries = index.EnumerateEntries().ToList(); + if (entries.Count == 0) continue; + + var samples = new[] { entries[0], entries[entries.Count / 2], entries[entries.Count - 1] }; + foreach (var sample in samples) + { + var target = PadKey(sample.KeyBytes, index.Header.KeyLength); + var results = index.Search(target).ToList(); + + results.ShouldContain(e => e.RecordNumber == sample.RecordNumber, $"tag: {tagName}"); + results.Count.ShouldBe( + entries.Count(e => PaddedEquals(e.KeyBytes, sample.KeyBytes, index.Header.KeyLength)), + $"tag: {tagName}"); + + searches++; + } + } + + searches.ShouldBeGreaterThan(0); + } + + [Theory] + [MemberData(nameof(CdxFixtures))] + public void Should_return_no_entries_for_missing_keys(string fixtureName) + { + using var cdxFile = OpenCdx(fixtureName); + + foreach (var tagName in cdxFile.TagNames) + { + var index = cdxFile.GetIndex(tagName); + if (index.Header.Order != CdxIndexOrder.Ascending) continue; + + var keyLength = index.Header.KeyLength; + var entries = index.EnumerateEntries().ToList(); + + var probes = new[] { CreateKey(0x01, keyLength), CreateKey(0xFE, keyLength) }; + foreach (var probe in probes) + { + if (entries.Any(e => PaddedEquals(e.KeyBytes, probe, keyLength))) continue; + + index.Search(probe).ShouldBeEmpty($"tag: {tagName}"); + } + } + } + + [Fact] + public void Should_find_dbf_rows_from_index_entries() + { + var fixtures = new[] + { + ("calls.CDX", "calls.dbf"), + ("contacts.CDX", "contacts.dbf"), + ("setup.CDX", "setup.dbf"), + ("types.CDX", "types.dbf") + }; + + var validatedTags = 0; + + foreach (var (cdxName, dbfName) in fixtures) + { + using var dbfTable = new DbfTable(Path.Combine(FixturesPath, dbfName), + stringTrimming: StringTrimmingOption.TrimEnd); + using var cdxFile = new CdxFile(Path.Combine(FixturesPath, cdxName), dbfTable.CurrentEncoding); + + var dbfRecord = new DbfRecord(dbfTable); + + foreach (var tagName in cdxFile.TagNames) + { + var index = cdxFile.GetIndex(tagName); + if (index.Header.Order != CdxIndexOrder.Ascending) continue; + + // only tags whose key expression is a plain character column can be + // validated against row values without evaluating index expressions + var column = dbfTable.Columns.FirstOrDefault(c => + string.Equals(c.ColumnName, index.KeyExpression, StringComparison.OrdinalIgnoreCase)); + if (column == null || column.ColumnType != DbfColumnType.Character) continue; + + var ordinal = dbfTable.Columns.IndexOf(column); + var entries = index.EnumerateEntries().ToList(); + var step = Math.Max(1, entries.Count / 100); + + for (var i = 0; i < entries.Count; i += step) + { + var entry = entries[i]; + + dbfTable.Seek(entry.RecordIndex); + dbfTable.Read(dbfRecord).ShouldBeTrue($"tag: {tagName}, record: {entry.RecordNumber}"); + + var value = dbfRecord.GetStringValue(ordinal) ?? string.Empty; + value.ShouldBe(entry.Key.TrimEnd(), $"tag: {tagName}, record: {entry.RecordNumber}"); + } + + validatedTags++; + } + } + + validatedTags.ShouldBeGreaterThan(0); + } + + private static CdxFile OpenCdx(string fixtureName) + { + return new CdxFile(Path.Combine(FixturesPath, fixtureName)); + } + + private static byte[] CreateKey(byte fill, int length) + { + var key = new byte[length]; + for (var i = 0; i < key.Length; i++) + { + key[i] = fill; + } + + return key; + } + + private static byte[] PadKey(byte[] key, int length) + { + if (key.Length == length) return key; + + var padded = CreateKey(0x20, length); + Array.Copy(key, padded, key.Length); + return padded; + } + + private static bool PaddedEquals(byte[] x, byte[] y, int length) + { + return ComparePadded(x, y, length) == 0; + } + + private static int ComparePadded(byte[] x, byte[] y, int length) + { + var paddedX = PadKey(x, length); + var paddedY = PadKey(y, length); + + for (var i = 0; i < length; i++) + { + var cmp = paddedX[i].CompareTo(paddedY[i]); + if (cmp != 0) return cmp; + } + + return 0; + } + } +}