Add Cortex.States.DuckDb: DuckDB-backed state store

Cortex.sln

@@ -72,6 +72,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Serialization", "Serializat
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Cortex.Streams.Mediator", "src\Cortex.Streams.Mediator\Cortex.Streams.Mediator.csproj", "{84410C57-0F59-F31F-B921-4C1F3D3FF144}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Cortex.States.DuckDb", "src\Cortex.States.DuckDb\Cortex.States.DuckDb.csproj", "{4FAE6C5E-53EE-4CCE-85A6-B7551A92C488}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -202,6 +204,10 @@ Global
 		{84410C57-0F59-F31F-B921-4C1F3D3FF144}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{84410C57-0F59-F31F-B921-4C1F3D3FF144}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{84410C57-0F59-F31F-B921-4C1F3D3FF144}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4FAE6C5E-53EE-4CCE-85A6-B7551A92C488}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4FAE6C5E-53EE-4CCE-85A6-B7551A92C488}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4FAE6C5E-53EE-4CCE-85A6-B7551A92C488}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4FAE6C5E-53EE-4CCE-85A6-B7551A92C488}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -233,6 +239,7 @@ Global
 		{44A166BD-01E9-4A4B-9BC5-7DE01B472E73} = {1C5D462D-168D-4D3F-B96E-CCE5517DB197}
 		{472BC645-9E2F-4205-A571-4D9184747EC5} = {7F9E0AEA-721E-46F8-90ED-8EA8423647FB}
 		{84410C57-0F59-F31F-B921-4C1F3D3FF144} = {4C68702C-1661-4AD9-83FD-E0B52B791969}
+		{4FAE6C5E-53EE-4CCE-85A6-B7551A92C488} = {C31F8C0F-8BCF-4959-9BA1-8645D058EAA0}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {E20303B6-8AC9-4FFF-B645-4608309ADA94}

README.md

@@ -114,6 +114,9 @@
 - **Cortex.States.SQLite:** Persistent state storage using SQLite.
 [![NuGet Version](https://img.shields.io/nuget/v/Cortex.States.SQLite?label=Cortex.States.SQLite)](https://www.nuget.org/packages/Cortex.States.SQLite)
 
+- **Cortex.States.DuckDb:** Persistent state storage using DuckDb.
+[![NuGet Version](https://img.shields.io/nuget/v/Cortex.States.DuckDb?label=Cortex.States.DuckDb)](https://www.nuget.org/packages/Cortex.States.DuckDb)
+
 - **Cortex.Telemetry:** Core library to add support for Tracing and Matrics.
 [![NuGet Version](https://img.shields.io/nuget/v/Cortex.Telemetry?label=Cortex.Telemetry)](https://www.nuget.org/packages/Cortex.Telemetry)
 

docs/Cortex.States.DuckDb.md

@@ -0,0 +1,310 @@
+# Cortex.States.DuckDb
+
+[![NuGet Version](https://img.shields.io/nuget/v/Cortex.States.DuckDb?label=Cortex.States.DuckDb)](https://www.nuget.org/packages/Cortex.States.DuckDb)
+
+**Cortex.States.DuckDb** is a state store implementation for the Cortex Data Framework that uses [DuckDB](https://duckdb.org/) as the underlying storage engine. DuckDB is an in-process analytical database management system designed for fast analytical queries, making it an excellent choice for scenarios requiring both transactional state management and analytical capabilities.
+
+## Features
+
+- **High-Performance Analytics**: Leverages DuckDB's columnar storage and vectorized query execution
+- **In-Memory & Persistent Storage**: Supports both in-memory databases for fast processing and file-based persistence
+- **Native Export Capabilities**: Export data directly to Parquet or CSV formats
+- **Batch Operations**: Efficient bulk insert and delete operations with transaction support
+- **Thread-Safe**: Built-in thread safety for concurrent access
+- **Flexible Serialization**: Customizable key and value serialization
+- **Fluent Builder API**: Easy configuration through builder pattern
+
+## Installation
+
+### Using the .NET CLI
+
+```bash
+dotnet add package Cortex.States.DuckDb
+```
+
+### Using the Package Manager Console
+
+```powershell
+Install-Package Cortex.States.DuckDb
+```
+
+## Quick Start
+
+### Basic Usage
+
+```csharp
+using Cortex.States.DuckDb;
+
+// Create a persistent DuckDB state store
+var stateStore = new DuckDbKeyValueStateStore<string, int>(
+    name: "MyStateStore",
+    databasePath: "./data/mystore.duckdb",
+    tableName: "KeyValueStore"
+);
+
+// Store values
+stateStore.Put("counter", 42);
+stateStore.Put("total", 100);
+
+// Retrieve values
+var counter = stateStore.Get("counter"); // Returns 42
+
+// Check if key exists
+if (stateStore.ContainsKey("counter"))
+{
+    Console.WriteLine("Counter exists!");
+}
+
+// Remove a value
+stateStore.Remove("counter");
+
+// Get all keys
+foreach (var key in stateStore.GetKeys())
+{
+    Console.WriteLine($"Key: {key}");
+}
+
+// Don't forget to dispose
+stateStore.Dispose();
+```
+
+### Using the Fluent Builder
+
+```csharp
+using Cortex.States.DuckDb;
+
+// Create store using fluent builder
+var stateStore = DuckDbKeyValueStateStoreBuilder<string, OrderSummary>
+    .Create("OrderStore")
+    .WithDatabasePath("./data/orders.duckdb")
+    .WithTableName("Orders")
+    .WithIndex(true)
+    .WithMaxMemory("2GB")
+    .WithThreads(4)
+    .Build();
+
+// Use the store
+stateStore.Put("ORD-001", new OrderSummary { Total = 99.99m, Status = "Completed" });
+```
+
+### In-Memory Database
+
+```csharp
+using Cortex.States.DuckDb;
+
+// Create an in-memory store for fast processing
+var inMemoryStore = DuckDbKeyValueStateStoreBuilder<string, decimal>
+    .Create("TemporaryStore")
+    .UseInMemory()
+    .WithTableName("TempData")
+    .Build();
+
+// Perfect for temporary computations
+inMemoryStore.Put("sum", 1234.56m);
+```
+
+### Using with Options
+
+```csharp
+using Cortex.States.DuckDb;
+
+// Create options for fine-grained control
+var options = new DuckDbKeyValueStateStoreOptions
+{
+    DatabasePath = "./data/analytics.duckdb",
+    TableName = "AnalyticsState",
+    CreateIndex = true,
+    MaxMemory = "4GB",
+    Threads = 8,
+    AccessMode = DuckDbAccessMode.ReadWrite
+};
+
+var stateStore = new DuckDbKeyValueStateStore<string, AnalyticsData>(
+    name: "AnalyticsStore",
+    options: options
+);
+```
+
+### Factory Methods
+
+```csharp
+using Cortex.States.DuckDb;
+
+// Quick creation methods
+var persistentStore = DuckDbStateStoreExtensions
+    .CreatePersistentDuckDbStore<string, Product>("ProductStore", "./data/products.duckdb", "Products");
+
+var inMemoryStore = DuckDbStateStoreExtensions
+    .CreateInMemoryDuckDbStore<string, Session>("SessionStore", "Sessions");
+```
+
+## Advanced Features
+
+### Batch Operations
+
+```csharp
+// Efficient bulk insert
+var items = new List<KeyValuePair<string, decimal>>
+{
+    new("price-1", 10.99m),
+    new("price-2", 20.99m),
+    new("price-3", 30.99m)
+};
+
+stateStore.PutMany(items);
+
+// Bulk delete
+stateStore.RemoveMany(new[] { "price-1", "price-2" });
+```
+
+### Export to Parquet/CSV
+
+DuckDB has native support for Parquet and CSV formats, making data export seamless:
+
+```csharp
+// Export to Parquet (ideal for analytics)
+stateStore.ExportToParquet("./exports/state-backup.parquet");
+
+// Export to CSV (ideal for data sharing)
+stateStore.ExportToCsv("./exports/state-backup.csv");
+```
+
+### Count and Clear
+
+```csharp
+// Get total count
+var count = stateStore.Count();
+Console.WriteLine($"Total items: {count}");
+
+// Clear all items
+stateStore.Clear();
+```
+
+### Checkpoint
+
+For persistent databases, you can force a checkpoint to ensure all data is written to disk:
+
+```csharp
+stateStore.Checkpoint();
+```
+
+## Integration with Cortex Streams
+
+Use DuckDB state store with Cortex Streams for stateful stream processing:
+
+```csharp
+using Cortex.Streams;
+using Cortex.States.DuckDb;
+
+// Create the state store
+var stateStore = new DuckDbKeyValueStateStore<string, int>(
+    name: "WordCountStore",
+    databasePath: "./data/wordcount.duckdb",
+    tableName: "WordCounts"
+);
+
+// Use in a stream pipeline
+var stream = StreamBuilder<string>.CreateNewStream("WordCountStream")
+    .Stream()
+    .FlatMap(line => line.Split(' '))
+    .GroupBy(word => word)
+    .Aggregate(
+        stateStore,
+        (count, word) => count + 1,
+        initialValue: 0)
+    .Sink(result => Console.WriteLine($"{result.Key}: {result.Value}"))
+    .Build();
+
+stream.Start();
+```
+
+## Custom Serialization
+
+You can provide custom serializers for complex types:
+
+```csharp
+using System.Text.Json;
+
+var stateStore = new DuckDbKeyValueStateStore<Guid, ComplexObject>(
+    name: "ComplexStore",
+    databasePath: "./data/complex.duckdb",
+    tableName: "ComplexData",
+    keySerializer: key => key.ToString(),
+    keyDeserializer: str => Guid.Parse(str),
+    valueSerializer: value => JsonSerializer.Serialize(value),
+    valueDeserializer: str => JsonSerializer.Deserialize<ComplexObject>(str)!
+);
+```
+
+## Configuration Options
+
+| Option | Description | Default |
+|--------|-------------|---------|
+| `DatabasePath` | Path to the DuckDB database file. Use `:memory:` for in-memory | Required |
+| `TableName` | Name of the table for key-value storage | Required |
+| `UseInMemory` | Use in-memory database instead of file | `false` |
+| `CreateIndex` | Create index on key column for faster lookups | `true` |
+| `MaxMemory` | Maximum memory limit (e.g., "1GB", "512MB") | Auto |
+| `Threads` | Number of threads (0 = auto) | `0` |
+| `AccessMode` | Database access mode (Automatic, ReadWrite, ReadOnly) | `Automatic` |
+
+## When to Use DuckDB State Store
+
+DuckDB is particularly well-suited for:
+
+- **Analytical workloads**: When you need to run analytical queries on your state
+- **Large datasets**: Efficient columnar storage for large amounts of data
+- **Data export requirements**: Native Parquet/CSV export capabilities
+- **Embedded analytics**: In-process database without external dependencies
+- **Temporary processing**: Fast in-memory mode for intermediate computations
+
+Consider other state stores when:
+
+- You need distributed state across multiple nodes (use Cassandra, MongoDB)
+- You require extreme write throughput (use RocksDB)
+- You need full ACID transactions across multiple operations (use PostgreSQL, SQL Server)
+
+## Thread Safety
+
+The `DuckDbKeyValueStateStore` is thread-safe and can be used concurrently from multiple threads. For in-memory databases, a persistent connection is maintained to ensure data consistency.
+
+## Error Handling
+
+```csharp
+try
+{
+    var value = stateStore.Get("non-existent-key");
+    if (value == null)
+    {
+        Console.WriteLine("Key not found");
+    }
+}
+catch (InvalidOperationException ex)
+{
+    Console.WriteLine($"Store not initialized: {ex.Message}");
+}
+```
+
+## Best Practices
+
+1. **Dispose properly**: Always dispose of the state store when done to release resources
+2. **Use batch operations**: For bulk inserts/deletes, use `PutMany` and `RemoveMany`
+3. **Choose appropriate storage**: Use in-memory for temporary data, file-based for persistence
+4. **Set memory limits**: Configure `MaxMemory` for large datasets to prevent excessive memory usage
+5. **Regular checkpoints**: Call `Checkpoint()` periodically for critical data in persistent mode
+
+## Requirements
+
+- .NET 7.0 or later
+- DuckDB.NET.Data package (automatically included)
+
+## License
+
+MIT License - see the [license file](../src/Cortex.States.DuckDb/Assets/license.md) for details.
+
+## Related Packages
+
+- [Cortex.States](https://www.nuget.org/packages/Cortex.States) - Core state management
+- [Cortex.States.RocksDb](https://www.nuget.org/packages/Cortex.States.RocksDb) - RocksDB state store
+- [Cortex.States.SQLite](https://www.nuget.org/packages/Cortex.States.SQLite) - SQLite state store
+- [Cortex.Streams](https://www.nuget.org/packages/Cortex.Streams) - Core streaming capabilities

src/Cortex.States.DuckDb/Assets/license.md

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Buildersoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.