Add the ordered-view incremental render engine that projects the grid, histogram, and status-bar presentation from a columnar store

Directory.Packages.props

@@ -8,6 +8,7 @@
   <ItemGroup>
     <PackageVersion Include="Microsoft.Maui.Controls" Version="$(MauiVersion)" />
     <PackageVersion Include="Microsoft.Maui.Controls.Compatibility" Version="$(MauiVersion)" />
+    <PackageVersion Include="Microsoft.AspNetCore.Components.Web" Version="10.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Components.WebView.Maui" Version="$(MauiVersion)" />
     <!-- Pinned to the version transitively resolved through Microsoft.Maui.Controls (per
              dotnet list package, include-transitive) so the ExplorerExtension companion exe and
@@ -30,6 +31,7 @@
          (dotnet/efcore PR 38402, milestone 11.0-preview6). Tracked by:
          https://github.com/microsoft/EventLogExpert/issues/604 -->
     <PackageVersion Include="SQLitePCLRaw.lib.e_sqlite3" Version="3.53.3" />
+    <PackageVersion Include="Fluxor" Version="6.10.0" />
     <PackageVersion Include="Fluxor.Blazor.Web" Version="6.10.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
     <PackageVersion Include="NSubstitute" Version="6.0.0" />

docs/Performance.md

@@ -60,12 +60,11 @@ newest rows stay as a bounded array-of-structs tail that seals into a new chunk
 target size, giving amortized O(1) append. Repeated strings fold into an interned pool and
 `EventData` field-name sequences into a shared schema table, and `Append` returns a new snapshot
 that reference-shares the prior chunks, pool, and schema table, so a published snapshot is safe to
-read while the next ingest builds. Each log exposes an `EventColumnView` that sorts its rows into a
-display order and reads fields straight through the store's reader with no event objects rehydrated;
-`CombinedColumnView` merges several such views with a K-way, column-direct cursor walk that compares
-head-to-head off each row's own reader, plus a periodic checkpoint index (stride 64) so a positional
-read seeks in log(rows) rather than walking from the start, with per-read offset scratch
-stack-allocated up to a capped K.
+read while the next ingest builds. The ordered-view engine maintains the display order
+incrementally in an `OrderedViewSnapshot` and reads fields straight through the store's reader with no
+event objects rehydrated; `OrderedColumnView` presents a single log and `CombinedOrderedColumnView`
+presents several, addressing each row to its owning reader over the pre-merged global order rather than
+re-running a K-way merge per read.
 
 ### P6 - Viewport virtualization and render-buffer reuse
 
@@ -109,7 +108,7 @@ filtering.
 | Read | Reverse read, newest bookmark, tuned `EvtNext` batch | `EventLogReader` |
 | Marshalling | Non-boxing packed `EventProperty` | `EventProperty`, `NativeMethods.Evt` |
 | Resolve | `ProcessorCount - 1` priority-gated parallelism | `OpenLogEffects` |
-| Store | Chunked columnar snapshot + K-way combined view | `EventColumnStore`, `EventColumnView`, `CombinedColumnView` |
+| Store | Chunked columnar snapshot + incrementally-ordered view engine | `EventColumnStore`, `OrderedColumnView`, `CombinedOrderedColumnView` |
 | Viewport | `Virtualize` + one-slice-per-window provider | `LogTablePane` |
 | Render | Per-thread grow-only render buffer, skip-probe | `NativeMethods.Evt` |
 | Filter memory | Retained structured `EventData` / `UserData` fields | `ResolvedEvent`, `UserDataValueExtractor` |

src/EventLogExpert.DatabaseTools/Common/Operations/OperationBase.cs

@@ -15,13 +15,13 @@ namespace EventLogExpert.DatabaseTools.Common.Operations;
 
 internal abstract class OperationBase
 {
+    protected static readonly TimeSpan DefaultFilterRegexTimeout = TimeSpan.FromSeconds(5);
     private static readonly string[] s_databaseFileSuffixes = ["", "-wal", "-shm"];
 
     private string _providerDetailFormat = "{0, -14} {1, 8} {2, 8} {3, 8} {4, 8} {5, 8} {6, 8}";
 
     public string? FailureSummary { get; private set; }
 
-    // Clearing SQLite pools releases Windows file handles before deleting the partial database.
     protected static async Task CleanupPartialDatabaseAsync(
         ITraceLogger logger,
         ProviderDbContext? dbContext,
@@ -54,7 +54,8 @@ protected static async Task CleanupPartialDatabaseAsync(
         }
     }
 
-    // Recompile infinite-timeout regexes so hostile patterns cannot hang the operation.
+    protected static Regex? EnsureBoundedTimeout(Regex? regex) => EnsureBoundedTimeout(regex, DefaultFilterRegexTimeout);
+
     protected static Regex? EnsureBoundedTimeout(Regex? regex, TimeSpan defaultTimeout)
     {
         if (regex is null) { return null; }
@@ -77,7 +78,6 @@ protected static async IAsyncEnumerable<ProviderDetails> LoadLocalProvidersAsync
         IReadOnlySet<string>? excludeProviderNames = null,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        // Async iterator bridge: synchronous provider reads share an IAsyncEnumerable consumer path.
         await Task.CompletedTask;
 
         foreach (var providerName in GetLocalProviderNames(regex))

src/EventLogExpert.DatabaseTools/CreateDatabase/CreateDatabaseOperation.cs

@@ -7,7 +7,6 @@
 using EventLogExpert.Eventing.OfflineImaging.VirtualDisk;
 using EventLogExpert.Eventing.OfflineImaging.Wim;
 using EventLogExpert.Eventing.OfflineImaging.Workspace;
-using EventLogExpert.Eventing.ProviderMetadata;
 using EventLogExpert.Logging.Abstractions;
 using EventLogExpert.Provider.Database.Context;
 using EventLogExpert.Provider.Database.Hashing;
@@ -21,12 +20,9 @@ internal sealed class CreateDatabaseOperation(CreateDatabaseRequest request) : O
 {
     private const int BatchSize = 100;
 
-    // SQLite at rest may include main, WAL, and SHM files; overwrite backup/restore must move all three together.
     private static readonly string[] s_databaseFileSuffixes = ["", "-wal", "-shm"];
 
-    // Set only after every sidecar backup moves; restore must not delete unmoved originals after a torn backup.
     private bool _overwriteBackupCompleted;
-
     private bool _overwriteBackupTaken;
 
     internal enum CreateDatabaseMode { Local, FileSource, OfflineImage }
@@ -43,7 +39,6 @@ public async Task<DatabaseToolsOutcome> ExecuteAsync(
             return DatabaseToolsOutcome.Failed;
         }
 
-        // Any stale .bak sidecar may be the sole surviving copy after an interrupted overwrite and must block retry.
         foreach (var suffix in s_databaseFileSuffixes)
         {
             var backupPath = request.TargetPath + suffix + ".bak";
@@ -69,7 +64,6 @@ public async Task<DatabaseToolsOutcome> ExecuteAsync(
             return DatabaseToolsOutcome.Failed;
         }
 
-        // Fail fast on destination ACL/CFA denial before expensive scan or extraction work begins.
         string targetDirectory = Path.GetDirectoryName(Path.GetFullPath(request.TargetPath)) ?? request.TargetPath;
         string? targetBlocked = OfflineScratch.ProbeWritable(targetDirectory);
 
@@ -98,7 +92,7 @@ public async Task<DatabaseToolsOutcome> ExecuteAsync(
             logger.Information($"{FormatSkippedProvidersMessage(excludeProviderNames.Count, request.SkipProvidersInFile)}");
         }
 
-        var filterRegex = EnsureBoundedTimeout(request.FilterRegex, TimeSpan.FromSeconds(5));
+        var filterRegex = EnsureBoundedTimeout(request.FilterRegex);
 
         var outcome = await CreateCoreAsync();
 
@@ -120,7 +114,6 @@ async Task<DatabaseToolsOutcome> CreateCoreAsync()
         var firstByIdentity = new Dictionary<ProviderIdentity, ProviderDetails>();
 #endif
 
-        // Create DbContext only after the first provider so failed scans leave no empty database.
         ProviderDbContext? dbContext = null;
         OfflineWimImage? wimImage = null;
         OfflineIsoImage? isoImage = null;
@@ -133,7 +126,6 @@ async Task<DatabaseToolsOutcome> CreateCoreAsync()
             string? effectiveOfflineImagePath = request.OfflineImagePath;
             OfflineImageKind? kind = mode == CreateDatabaseMode.OfflineImage ? ResolveImageKind(request) : null;
 
-            // WIM apply ignores cooperative cancellation on denied writes, so probe scratch ACLs before native extraction.
             if (kind is OfflineImageKind.Wim or OfflineImageKind.Iso)
             {
                 string? scratchBlocked = OfflineScratch.ProbeWritable(OfflineScratch.Root);
@@ -191,7 +183,6 @@ async Task<DatabaseToolsOutcome> CreateCoreAsync()
                 effectiveOfflineImagePath = wimImage!.ExtractedRoot;
             }
 
-            // Offline providers already carry image provenance; only local builds read host provenance.
             IAsyncEnumerable<ProviderDetails> providersToAdd;
             SourceOsProvenance? sourceOsProvenance;
 
@@ -335,13 +326,11 @@ async Task<DatabaseToolsOutcome> CreateCoreAsync()
         {
             if (dbContext is not null) { await dbContext.DisposeAsync(); }
 
-            // Dispose extracted WIM after SaveChanges because persisted rows may still read from it.
             wimImage?.Dispose();
             isoImage?.Dispose();
             vhdxImage?.Dispose();
         }
 
-        // Never delete target files after a torn overwrite backup; they may be unmoved originals.
         async Task CleanupPartialUnlessUnmovedOriginalAsync()
         {
             if (_overwriteBackupTaken && !_overwriteBackupCompleted) { return; }
@@ -350,12 +339,10 @@ async Task CleanupPartialUnlessUnmovedOriginalAsync()
         }
         }
 
-        // Back up the old database before the writable context opens or creates target files.
         ProviderDbContext GetOrCreateContext()
         {
             if (request.Overwrite && !_overwriteBackupTaken && File.Exists(request.TargetPath))
             {
-                // Set before moving so a torn backup still enters restore.
                 _overwriteBackupTaken = true;
                 TakeOverwriteBackup();
                 _overwriteBackupCompleted = true;
@@ -383,10 +370,8 @@ internal static CreateDatabaseMode SelectMode(CreateDatabaseRequest request) =>
 
     internal static bool ValidateOfflineImageRequest(CreateDatabaseRequest request, ITraceLogger logger)
     {
-        // Kind-ambiguous validation errors are attributed to the Offline root; each resolved kind uses its fine category.
         ITraceLogger offlineLogger = logger.ForCategory(LogCategories.Offline);
 
-        // Reject orphan WIM options so the command cannot silently fall back to local providers.
         if (string.IsNullOrWhiteSpace(request.OfflineImagePath))
         {
             if (request.ImageKind is not null)
@@ -487,7 +472,6 @@ internal static bool ValidateOfflineImageRequest(CreateDatabaseRequest request,
                     return false;
                 }
 
-                // Validator cannot mount ISO just to list choices; extraction reports bad indices after mount.
                 if (request.WimIndex is null)
                 {
                     isoLogger.Error(
@@ -693,7 +677,6 @@ private async Task FlushHeaderAndBufferAsync(
         buffer.Clear();
     }
 
-    // Clear pools and remove aborted-build sidecars before restoring backups.
     private void RestoreOverwriteBackups(ITraceLogger logger)
     {
         var mainBackup = request.TargetPath + ".bak";
@@ -702,14 +685,12 @@ private void RestoreOverwriteBackups(ITraceLogger logger)
         {
             SqliteConnection.ClearAllPools();
 
-            // Only delete files after a complete backup; otherwise they may be unmoved originals.
             if (_overwriteBackupCompleted)
             {
                 foreach (var suffix in s_databaseFileSuffixes)
                 {
                     var newFile = request.TargetPath + suffix;
 
-                    // Never delete the main file unless its snapshot exists to take its place.
                     if (suffix.Length == 0 && !File.Exists(mainBackup)) { continue; }
 
                     if (File.Exists(newFile)) { File.Delete(newFile); }
@@ -731,7 +712,6 @@ private void RestoreOverwriteBackups(ITraceLogger logger)
         }
     }
 
-    // Stale-backup preflight guarantees every .bak move targets a free path.
     private void TakeOverwriteBackup()
     {
         foreach (var suffix in s_databaseFileSuffixes)
@@ -815,7 +795,6 @@ private static bool ModelsEquivalent<TModel, TIdentity>(
         Func<TModel, TModel, bool> areEquivalent)
         where TIdentity : notnull
     {
-        // Compare distinct identities both ways because the hash drops exact duplicate rows.
         var firstByIdentity = new Dictionary<TIdentity, TModel>(first.Count);
 
         foreach (TModel model in first) { firstByIdentity[identityOf(model)] = model; }

src/EventLogExpert.DatabaseTools/ShowProviders/ShowProvidersOperation.cs

@@ -8,10 +8,6 @@
 
 namespace EventLogExpert.DatabaseTools.ShowProviders;
 
-/// <summary>
-///     Lists provider details from either local providers (<see cref="ShowProvidersRequest.SourcePath" /> = null) or
-///     a specified source (.db / .evtx / folder). Streams output as each provider is resolved.
-/// </summary>
 internal sealed class ShowProvidersOperation(ShowProvidersRequest request) : OperationBase, IDatabaseToolsOperation
 {
     private const int HeaderBatchSize = 100;
@@ -21,8 +17,7 @@ public async Task<DatabaseToolsOutcome> ExecuteAsync(
         IProgress<DatabaseToolsProgress>? progress,
         CancellationToken cancellationToken)
     {
-        // Defensive recompile if input has Regex.InfiniteMatchTimeout (otherwise catch below is dead).
-        var filterRegex = EnsureBoundedTimeout(request.FilterRegex, TimeSpan.FromSeconds(5));
+        var filterRegex = EnsureBoundedTimeout(request.FilterRegex);
 
         // Buffer first batch to size the column widths (mirrors CreateDatabaseOperation). Lives outside
         // the try so the cancellation arm can flush partial output the user has been waiting on.

src/EventLogExpert.ElevationHelper/ProgramEntry.cs

@@ -16,10 +16,12 @@ namespace EventLogExpert.ElevationHelper;
 internal static class ProgramEntry
 {
     private const int CancelWatchdogExitCode = 12;
-    // High-IL helper must kill itself if native code ignores cooperative cancellation; the medium-IL host cannot.
-    private static readonly TimeSpan s_selfTerminateWatchdog = TimeSpan.FromSeconds(8);
+    private const int PipeConnectTimeoutMilliseconds = 10_000;
 
-    // Startup orphan cleanup must not block the control reader from receiving cancellation.
+    private static readonly TimeSpan s_controlReaderDrainTimeout = TimeSpan.FromSeconds(1);
+    private static readonly TimeSpan s_requestReadTimeout = TimeSpan.FromSeconds(10);
+    private static readonly TimeSpan s_selfTerminateWatchdog = TimeSpan.FromSeconds(8);
+    private static readonly TimeSpan s_selfTerminateWriteTimeout = TimeSpan.FromSeconds(2);
     private static readonly TimeSpan s_startupReconcileTimeout = TimeSpan.FromSeconds(30);
 
     public static async Task<int> MainAsync(string[] args)
@@ -55,7 +57,7 @@ private static async Task<int> RunOperationModeAsync(IpcMessageReader reader, Ip
 
         try
         {
-            using var requestCts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+            using var requestCts = new CancellationTokenSource(s_requestReadTimeout);
             request = await reader.ReadRequestAsync(requestCts.Token);
         }
         catch (OperationCanceledException)
@@ -80,7 +82,6 @@ private static async Task<int> RunOperationModeAsync(IpcMessageReader reader, Ip
 
         using var operationCts = new CancellationTokenSource();
 
-        // Armed on CancelMessage and cancelled on completion so only an ignored cooperative cancel self-terminates.
         using var watchdogCts = new CancellationTokenSource();
 
         var controlReaderTask = Task.Run(async () =>
@@ -104,7 +105,6 @@ private static async Task<int> RunOperationModeAsync(IpcMessageReader reader, Ip
                     {
                         operationCts.Cancel();
 
-                        // If the operation does not unwind, self-terminate so the host observes pipe EOF promptly.
                         try { await Task.Delay(s_selfTerminateWatchdog, watchdogCts.Token); }
                         catch (OperationCanceledException) { return; }
 
@@ -121,7 +121,6 @@ private static async Task<int> RunOperationModeAsync(IpcMessageReader reader, Ip
             }
         });
 
-        // Run orphan cleanup off the main flow so a wedged delete cannot block cancellation handling.
         var reconcileTask = Task.Run(() =>
         {
             try { OfflineMaintenance.ReconcileOrphans(logger: null); }
@@ -141,7 +140,7 @@ private static async Task<int> RunOperationModeAsync(IpcMessageReader reader, Ip
         {
             watchdogCts.Cancel();
             operationCts.Cancel();
-            try { await controlReaderTask.WaitAsync(TimeSpan.FromSeconds(1)); } catch { /* Best-effort control-reader drain. */ }
+            try { await controlReaderTask.WaitAsync(s_controlReaderDrainTimeout); } catch { /* Best-effort control-reader drain. */ }
 
             await TryWriteTerminalAsync(writer,
                 new FatalMessage(ex.GetType().FullName ?? ex.GetType().Name, ex.Message, ex.StackTrace ?? string.Empty));
@@ -152,7 +151,7 @@ await TryWriteTerminalAsync(writer,
         watchdogCts.Cancel();
         operationCts.Cancel();
 
-        try { await controlReaderTask.WaitAsync(TimeSpan.FromSeconds(1)); } catch { /* Best-effort control-reader drain. */ }
+        try { await controlReaderTask.WaitAsync(s_controlReaderDrainTimeout); } catch { /* Best-effort control-reader drain. */ }
 
         await TryWriteTerminalAsync(writer,
             new ResultMessage(result.Outcome, result.FailureSummary, (long)result.Duration.TotalMilliseconds));
@@ -166,12 +165,11 @@ private static async Task<int> RunPipeModeAsync(string pipeName, bool probe)
             ".",
             pipeName,
             PipeDirection.InOut,
-            // Omit client-side CurrentUserOnly: elevated and medium-IL same-user tokens have different SIDs; server DACL plus PID verification authenticates.
             PipeOptions.Asynchronous);
 
         try
         {
-            await pipe.ConnectAsync(timeout: 10_000);
+            await pipe.ConnectAsync(timeout: PipeConnectTimeoutMilliseconds);
         }
         catch (TimeoutException)
         {
@@ -208,15 +206,14 @@ await writer.WriteAsync(
         }
     }
 
-    // Last-resort self-termination leaves orphaned mounts for next elevated-launch reconciliation.
     private static async Task SelfTerminateAfterUnresponsiveCancelAsync(IpcMessageWriter writer, long elapsedMs)
     {
         try
         {
             await TryWriteTerminalAsync(writer, new ResultMessage(
                 DatabaseToolsOutcome.Cancelled,
                 "Cancelled; the elevated helper self-terminated after a native operation ignored cancellation.",
-                elapsedMs)).WaitAsync(TimeSpan.FromSeconds(2));
+                elapsedMs)).WaitAsync(s_selfTerminateWriteTimeout);
         }
         catch { /* Pipe EOF on exit is the guaranteed host signal. */ }
 

src/EventLogExpert.EventDbTool/EventLogExpert.EventDbTool.csproj

@@ -3,7 +3,6 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <RuntimeIdentifiers>win-x64</RuntimeIdentifiers>
-    <PlatformTarget>x64</PlatformTarget>
     <AssemblyName>eventdbtool</AssemblyName>
   </PropertyGroup>