From c812fd7de2473df047e47a15a1cf4c3109dd957c Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Wed, 3 Sep 2025 13:54:09 +0100 Subject: [PATCH 01/22] Add proposal for optimising Dynamo DB outbox usage --- ...-reads-and-writes-from-dynamo-outbox.md.md | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md diff --git a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md new file mode 100644 index 0000000000..9940ed32b5 --- /dev/null +++ b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md @@ -0,0 +1,91 @@ +# 33. Optimise reads and write from Dynamo outbox + +Date: 2025-09-03 + +## Status + +Proposed + +## Context + +Load testing of APIs using Brighter with a Dynamo DB outbox has shown a performance bottleneck in high throughput scenarios, resulting in excessive CPU usage, high response times, and some HTTP 503 responses to clients. When running load testing locally with the API hooked up to LocalStack, we see an excessive number of `Query` operations being performed on the Dynamo outbox table. This suggests that the issue is some form of resource exhaustion cause by excessive requests being made over the network. + +There are several places where we make inefficient use of the Dynamo DB client. + +### `OutstandingMessages` operation + +The `Outstanding` GSI in the Dynamo outbox uses the topic name as a primary key, which is then sharded according to the number of shards provided in config by the user in order to avoid hot partitions. When the outbox is queried to fetch all outstanding messages, it performs a large number of queries iterating over each topic it knows about, and each shard for each of those partitions. If the outbox was being used as part of publishing to five topics, each with 20 shards in the outbox table, that would mean performing 100 query operations even if the outstanding index was completely empty. + +### `DispatchedMessages` operation & the `Delivered` GSI + +Similar to the operation above. The `Delivered` index is partitioned by topic name, but unlike the `Outstanding` index it is _not_ sharded. This means that when the `DispatchedMessage` operation is performed, it only has to iterate over the topics it knows about (so following the scenario above, five queries even if the delivered index is empty). + +As the `Delivered` index isn't sharded, it can fall victim to hot partitions. When writes to a GSI are throttled, [this also throttles writes to the base table that would affect the GSI](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/gsi-throttling.html). Since the `Delivered` index is a sparse index that isn't written to if the `DeliveryTime` of a message is null, this wouldn't affect the initial writing of messages to the outbox. It _would_, however, throttle the ability to mark messages as delivered, and lead to duplicate publishes as messages become "stuck" in the outbox. + +### Batch get and write operations + +There are a few operations where the outbox is provided with a collection of messages or message IDs, and instead of performing batch operations with those IDs it iterates through them and performs individual operations sequentially: + +* When clearing a collection of message IDs from the outbox, it fetches each of these individually and tries to dispatch it before moving onto the next +* When marking a collection of messages as dispatched, the messages are written to sequentially with seperate requests +* When a collection of messages are deleted from the outbox they're worked through sequentially with separate requests + +### Fetching outstanding message count + +Every time one or more messages are cleared from the outbox, the `OutboxProducerMediator` checks whether it needs to update it's internal metric for how many outstanding messages currently sit in the outbox, based on when that last check was last run. If it determines a refresh is neccesary, it fetches all outstanding messages from the outbox into memory (but does so asynchronously). This comes with all the queries described above, and if there are a large number of messages outstanding, can eat up a problematic amount of memory. The count of outstanding messages only appears to be used for monitoring purposes. + +## Decision + +All of the inefficiencies above can be improved with non-breaking changes. + +### `OutstandingMessages` operation + +Given that the `Outstanding` index is a sparse index, and we wish to pull out the entirety of that index when we perform the operation, this can be a `Scan` operation on the index instead of a `Query`. This removes the need to iterate over topics and shards, and can instead be done as a single HTTP call if the number of outstanding messages allows it (with paging if it doesn't). + +The one downside of this is that we cannot specify the ordering of results from a `Scan` operation. If the results are paged, we will not be able to specify that the oldest messages should be retrieved first. Given the performance issues using `Query` operations, and the limitations of Dynamo DB as a storage platform, this feels like a reasonable comprompise to make. + +If a specific topic is provided in the `args` dictionary when performing the `OutstandingMessages` operation, then a `Query` would still need to be used, iterating over the shards for that topic. + +### `DispatchedMessages` operation + +As above - if no topic is provided in the `args` dictionary, use a `Scan` operation to fetch dispatched messages instead of a `Query` operation. + +### The `Delivered` GSI + +Introduce sharding to the `Delivered` index, using the same number of shards as configured for the `Outstanding` index. + +### Getting messages for dispatch + +Add overloads of both `Get` and `GetAsync` to the outbox interfaces that take a collection of message IDs instead of just one. For the Dynamo DB implementation, use a `BatchGet` operation to fetch all of them at once. For the other implementations, they can just iterate over their other `Get` methods for now. + +If the `BatchGet` operation only returns a subset of the requested messages, throw an exception. + +Update `OutboxProducerMediator` to use the new `Get` methods. + +### Marking messages as dispatched + +When marking a collection of messages as dispatched, us a `BatchWrite` operation to update all of them at once. If any of the updates fail, throw an exception. + +Add a new overload of `MarkDispatched` which takes a collection of message IDs (currently the only bulk option is the async version). + +### Deleting messages from the outbox + +When deleting a collection of messages from the outbox, do so using a `BatchWrite` operation. If any of the deletes fail, throw an exception. + +### Outstanding item count + +It feels useful to have the number of outstanding messages available in a metric. Dynamo DB doesn't have a `Count` operation, but we can get this information without having to pull all messages into memory: + +1. When messages are added to the Dynamo DB outbox, add a new binary attribute containing a single bit of data. +2. Add new methods to the outbox interfaces for `GetOutstandingMessageCount` and `GetOutstandingMessageCountAsync`. +3. In the Dynamo DB implementation, perform a `Scan` operation on the `Outstanding` index, with configuration to only retrieve the new binary attribute. +4. Page through results as required, and sum the total number of records returned. + +This minimises the amount of data sent over the wire, minimises memory consumption, and maximises the number of records returned in each page. + +Other outbox implementations can continue to use their implementations of `OutstandingMessages` for now. + +## Consequences + +* We will no longer be able to sort results of the `OutstandingMessages` or `DispatchedMessages` operations (when performed for all topics) to ensure the retrieval of oldest messages first from a Dynamo DB outbox, but will support high throughput scenarios instead. The results can still be sorted if queried on a topic by topic basis. +* The possibility of future improvements to other outbox implementations, to take advantage of the new bulk operation methods \ No newline at end of file From e1577c84d60f43f9abf6939acfcfbb79f0ba2e43 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Wed, 3 Sep 2025 13:55:36 +0100 Subject: [PATCH 02/22] Improve ADR title --- .../adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md index 9940ed32b5..c24a2eb1a5 100644 --- a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md +++ b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md @@ -1,4 +1,4 @@ -# 33. Optimise reads and write from Dynamo outbox +# 33. Optimise reads/writes from/to Dynamo outbox Date: 2025-09-03 From 6c40914e1b81c49c917961fee6a4977e017c3758 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Thu, 11 Sep 2025 11:17:08 +0100 Subject: [PATCH 03/22] Update ADR with decision on Dynamo DB optimisation --- ...se-reads-and-writes-from-dynamo-outbox.md} | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) rename docs/adr/{0033-optimise-reads-and-writes-from-dynamo-outbox.md.md => 0033-optimise-reads-and-writes-from-dynamo-outbox.md} (66%) diff --git a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md similarity index 66% rename from docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md rename to docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md index c24a2eb1a5..667ec7f7ee 100644 --- a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md.md +++ b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md @@ -40,15 +40,23 @@ All of the inefficiencies above can be improved with non-breaking changes. ### `OutstandingMessages` operation -Given that the `Outstanding` index is a sparse index, and we wish to pull out the entirety of that index when we perform the operation, this can be a `Scan` operation on the index instead of a `Query`. This removes the need to iterate over topics and shards, and can instead be done as a single HTTP call if the number of outstanding messages allows it (with paging if it doesn't). +Introduce a new additional `Outstanding` GSI called `OutstandingAllTopics`. This index will use `OutstandingCreatedTime` as the hash key, and `MessageId` as the range key, making it a sparse, high cardinality index containing outstanding messages for all topics. -The one downside of this is that we cannot specify the ordering of results from a `Scan` operation. If the results are paged, we will not be able to specify that the oldest messages should be retrieved first. Given the performance issues using `Query` operations, and the limitations of Dynamo DB as a storage platform, this feels like a reasonable comprompise to make. +When requests are made to the outbox to fetch outstanding messages for all topics, scan the new index using a parallel `Scan` and fetch results up to the provided page size. Order the results by created time in memory before returning them to the calling function. -If a specific topic is provided in the `args` dictionary when performing the `OutstandingMessages` operation, then a `Query` would still need to be used, iterating over the shards for that topic. +Add a new configuration option to `DynamoDbConfiguration` called `ScanConcurrency` to allow configurability of how many parallel scan operations are performed concurrently. + +When requests are made to the outbox to fetch outstanding message for a specific topic, continue to use a `Query` operation on the existing index and iterate through shards, fetching results up to the page size. + +The one downside of this is that we cannot specify the ordering of results from a `Scan` operation. We try to get around this by ordering the results in memory, but if the number of outstanding messages in the outbox is greater than the page size, the ordering of messages returned by the operation cannot be guaranteed. ### `DispatchedMessages` operation -As above - if no topic is provided in the `args` dictionary, use a `Scan` operation to fetch dispatched messages instead of a `Query` operation. +As above: + + * Introduce a new `Delivered` index called `DeliveredAllTopics`, which uses `DeliveryTime` as the hash key and `MessageId` as the range key + * Scan the new index when fetching delivered messages for all topics, using the `ScanConcurrency` option for parallel scan concurrency + * Continue to use a `Query` operation when fetching delivered messages for a specific topic, iterating through shards ### The `Delivered` GSI @@ -64,7 +72,9 @@ Update `OutboxProducerMediator` to use the new `Get` methods. ### Marking messages as dispatched -When marking a collection of messages as dispatched, us a `BatchWrite` operation to update all of them at once. If any of the updates fail, throw an exception. +When marking a collection of messages as dispatched, use a `BatchWrite` operation to update all of them at once. + +By default, if any of the updates fail throw an exception. If, however, an option is passed in the `args` dictionary with the key `LogBatchErrors` and a value of `true`, then don't throw and instead log a warning message. When the method is invoked from the outbox sweeper, pass this new option to the method. Add a new overload of `MarkDispatched` which takes a collection of message IDs (currently the only bulk option is the async version). @@ -85,7 +95,21 @@ This minimises the amount of data sent over the wire, minimises memory consumpti Other outbox implementations can continue to use their implementations of `OutstandingMessages` for now. +### Deterministic shard assignment + +Make assignment of messages to shards for each topic deterministic. This makes it possible to preserve ordering of messages within a partition key by ensuring all messages with that key are assigned to the same shard. This can be done by hashing the partition key on the message: + +```c# +var keyBytes = Encoding.UTF8.GetBytes(message.Header.PartitionKey); +var sha256 = SHA256.Create(); +var keyHash = sha256.ComputeHash(keyBytes); +var shardNumber = BitConverter.ToUInt32(keyHash, 0) % _configuration.NumberOfShards; +``` + +If the partition key isn't specified for a message, then fall back to random shard assignment. + ## Consequences -* We will no longer be able to sort results of the `OutstandingMessages` or `DispatchedMessages` operations (when performed for all topics) to ensure the retrieval of oldest messages first from a Dynamo DB outbox, but will support high throughput scenarios instead. The results can still be sorted if queried on a topic by topic basis. +* When performing the `OutstandingMessages` or `DispatchedMessages` operations for all topics, we will only be able to guarantee the order of the returned messages if the number of outstanding messages is less than the page size for the operation. +* Shards will be assigned to messages deterministically based on their partition key * The possibility of future improvements to other outbox implementations, to take advantage of the new bulk operation methods \ No newline at end of file From 3b149e846c3e3df11175a70c2c51fc801e5651a9 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Thu, 11 Sep 2025 15:15:16 +0100 Subject: [PATCH 04/22] Scan outbox over all topics This commit adds two new indices to the Dynamo DB outbox structure, allowing for effective scanning of all outstanding or delivered messages regardless of topic. --- Brighter.sln | 40 ++-- ...ueryContext.cs => AllTopicsScanContext.cs} | 5 +- .../DynamoDbConfiguration.cs | 24 ++- .../DynamoDbOutbox.cs | 186 ++++++++---------- .../MessageItem.cs | 5 +- .../OutstandingAllTopicsQueryContext.cs | 44 ----- 6 files changed, 126 insertions(+), 178 deletions(-) rename src/Paramore.Brighter.Outbox.DynamoDB/{DispatchedAllTopicsQueryContext.cs => AllTopicsScanContext.cs} (84%) delete mode 100644 src/Paramore.Brighter.Outbox.DynamoDB/OutstandingAllTopicsQueryContext.cs diff --git a/Brighter.sln b/Brighter.sln index b127e5f32e..400fbc585b 100644 --- a/Brighter.sln +++ b/Brighter.sln @@ -2046,6 +2046,18 @@ Global {5D57A811-A3C2-42BA-A086-A5BFE8B590C7}.Release|Mixed Platforms.Build.0 = Release|Any CPU {5D57A811-A3C2-42BA-A086-A5BFE8B590C7}.Release|x86.ActiveCfg = Release|Any CPU {5D57A811-A3C2-42BA-A086-A5BFE8B590C7}.Release|x86.Build.0 = Release|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|x86.ActiveCfg = Debug|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|x86.Build.0 = Debug|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Release|Any CPU.Build.0 = Release|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Release|x86.ActiveCfg = Release|Any CPU + {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Release|x86.Build.0 = Release|Any CPU {022AD920-4E8D-4370-9C6D-CA4D8DA3DB6F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {022AD920-4E8D-4370-9C6D-CA4D8DA3DB6F}.Debug|Any CPU.Build.0 = Debug|Any CPU {022AD920-4E8D-4370-9C6D-CA4D8DA3DB6F}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -2274,6 +2286,18 @@ Global {D530B147-067A-408D-BB1B-A4290324012F}.Release|Mixed Platforms.Build.0 = Release|Any CPU {D530B147-067A-408D-BB1B-A4290324012F}.Release|x86.ActiveCfg = Release|Any CPU {D530B147-067A-408D-BB1B-A4290324012F}.Release|x86.Build.0 = Release|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Any CPU.Build.0 = Debug|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|x86.ActiveCfg = Debug|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|x86.Build.0 = Debug|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Any CPU.ActiveCfg = Release|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Any CPU.Build.0 = Release|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|x86.ActiveCfg = Release|Any CPU + {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|x86.Build.0 = Release|Any CPU {7AA5B0BF-3520-45C4-9B8A-7F131EFDA227}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7AA5B0BF-3520-45C4-9B8A-7F131EFDA227}.Debug|Any CPU.Build.0 = Debug|Any CPU {7AA5B0BF-3520-45C4-9B8A-7F131EFDA227}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -2406,18 +2430,6 @@ Global {3B6D084F-C034-49C6-A8C4-3C23DCC83CF2}.Release|Mixed Platforms.Build.0 = Release|Any CPU {3B6D084F-C034-49C6-A8C4-3C23DCC83CF2}.Release|x86.ActiveCfg = Release|Any CPU {3B6D084F-C034-49C6-A8C4-3C23DCC83CF2}.Release|x86.Build.0 = Release|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Any CPU.Build.0 = Debug|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|x86.ActiveCfg = Debug|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Debug|x86.Build.0 = Debug|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Any CPU.ActiveCfg = Release|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Any CPU.Build.0 = Release|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|x86.ActiveCfg = Release|Any CPU - {62D9AF6E-B671-4444-82A4-8416B8049E14}.Release|x86.Build.0 = Release|Any CPU {4EA5F196-DDA8-4941-956B-D413B03051C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4EA5F196-DDA8-4941-956B-D413B03051C9}.Debug|Any CPU.Build.0 = Debug|Any CPU {4EA5F196-DDA8-4941-956B-D413B03051C9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -2490,8 +2502,6 @@ Global {0E6A0B80-58B7-4AA2-9E40-EE0AA5D4719E}.Release|Mixed Platforms.Build.0 = Release|Any CPU {0E6A0B80-58B7-4AA2-9E40-EE0AA5D4719E}.Release|x86.ActiveCfg = Release|Any CPU {0E6A0B80-58B7-4AA2-9E40-EE0AA5D4719E}.Release|x86.Build.0 = Release|Any CPU - {F00B137A-C187-4C33-A37B-22AD40B71600}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F00B137A-C187-4C33-A37B-22AD40B71600}.Debug|Any CPU.Build.0 = Debug|Any CPU {79CA356E-B08C-4D88-88C9-653EC8D8BF4D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {79CA356E-B08C-4D88-88C9-653EC8D8BF4D}.Debug|Any CPU.Build.0 = Debug|Any CPU {79CA356E-B08C-4D88-88C9-653EC8D8BF4D}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU @@ -2528,8 +2538,6 @@ Global {24360989-A956-45E9-BF07-7FD9E7553C7D}.Release|Mixed Platforms.Build.0 = Release|Any CPU {24360989-A956-45E9-BF07-7FD9E7553C7D}.Release|x86.ActiveCfg = Release|Any CPU {24360989-A956-45E9-BF07-7FD9E7553C7D}.Release|x86.Build.0 = Release|Any CPU - {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9D7DFB30-1E56-46C7-A56E-2FBC6C58EB96}.Debug|Any CPU.Build.0 = Debug|Any CPU {9063F17B-5636-4AD5-999B-C894517DB5FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9063F17B-5636-4AD5-999B-C894517DB5FD}.Debug|Any CPU.Build.0 = Debug|Any CPU {9063F17B-5636-4AD5-999B-C894517DB5FD}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DispatchedAllTopicsQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB/AllTopicsScanContext.cs similarity index 84% rename from src/Paramore.Brighter.Outbox.DynamoDB/DispatchedAllTopicsQueryContext.cs rename to src/Paramore.Brighter.Outbox.DynamoDB/AllTopicsScanContext.cs index d8d1d0e281..980b2fee3d 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DispatchedAllTopicsQueryContext.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/AllTopicsScanContext.cs @@ -23,13 +23,10 @@ THE SOFTWARE. */ #endregion -using System.Collections.Generic; - namespace Paramore.Brighter.Outbox.DynamoDB; -internal sealed class DispatchedAllTopicsQueryContext(int nextPage, string? lastEvaluatedKey, List remainingTopics) +internal sealed class AllTopicsScanContext(int nextPage, string? lastEvaluatedKey) { public int NextPage { get; private set; } = nextPage; public string? LastEvaluatedKey { get; private set; } = lastEvaluatedKey; - public List RemainingTopics { get; private set; } = remainingTopics; } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs index eff7e012d8..29077f876a 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs @@ -3,21 +3,31 @@ namespace Paramore.Brighter.Outbox.DynamoDB { public class DynamoDbConfiguration - { + { /// /// The table that forms the Outbox /// public string TableName { get; set; } - + /// - /// The name of local delivered status index + /// The name of the global secondary delivered index indexed by topic /// - public string DeliveredIndexName { get; } - + public string DeliveredIndexName { get; set; } + + /// + /// The name of the global secondary delivered index convering all topics + /// + public string DeliveredAllTopicsIndexName { get; set; } + /// - /// The name of the global secondary outstanding index + /// The name of the global secondary outstanding index indexed by topic /// public string OutstandingIndexName { get; set; } + + /// + /// The name of the global secondary outstanding index convering all topics + /// + public string OutstandingAllTopicsIndexName { get; set; } /// /// Timeout in milliseconds @@ -45,7 +55,9 @@ public DynamoDbConfiguration(string? tableName = null, int timeout = 500, int nu { TableName = tableName ?? "brighter_outbox"; OutstandingIndexName = "Outstanding"; + OutstandingAllTopicsIndexName = "OutstandingAllTopics"; DeliveredIndexName = "Delivered"; + DeliveredAllTopicsIndexName = "DeliveredAllTopics"; Timeout = timeout; NumberOfShards = numberOfShards; } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 5c6173707b..2ebff4b197 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -27,7 +27,6 @@ THE SOFTWARE. */ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Amazon.DynamoDBv2; @@ -52,10 +51,8 @@ public class DynamoDbOutbox : private readonly ConcurrentDictionary _outstandingTopicQueryContexts; private readonly ConcurrentDictionary _dispatchedTopicQueryContexts; - private readonly ConcurrentDictionary _topicNames; - - private OutstandingAllTopicsQueryContext? _outstandingAllTopicsQueryContext; - private DispatchedAllTopicsQueryContext? _dispatchedAllTopicsQueryContext; + private AllTopicsScanContext? _outstandingAllTopicsScanContext; + private AllTopicsScanContext? _dispatchedAllTopicsScanContext; private readonly InstrumentationOptions _instrumentationOptions; private const string DYNAMO_DB_NAME = "outbox"; @@ -98,7 +95,6 @@ public DynamoDbOutbox( _outstandingTopicQueryContexts = new ConcurrentDictionary(); _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - _topicNames = new ConcurrentDictionary(); _instrumentationOptions = instrumentationOptions; } @@ -122,7 +118,6 @@ public DynamoDbOutbox(DynamoDBContext context, DynamoDbConfiguration configurati _outstandingTopicQueryContexts = new ConcurrentDictionary(); _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - _topicNames = new ConcurrentDictionary(); } /// @@ -194,9 +189,6 @@ public async Task AddAsync( var expiresAt = GetExpirationTime(); var messageToStore = new MessageItem(message, shard, expiresAt); - // Store the name of the topic as a key in a concurrent dictionary to ensure uniqueness & thread safety - _topicNames.TryAdd(message.Header.Topic, 0); - if (transactionProvider != null) { await AddToTransactionWrite(messageToStore, (DynamoDbUnitOfWork)transactionProvider); @@ -590,73 +582,59 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1 && _outstandingAllTopicsQueryContext?.NextPage != pageNumber) + if (pageNumber != 1 && _outstandingAllTopicsScanContext?.NextPage != pageNumber) { - var nextPageNumber = _outstandingAllTopicsQueryContext?.NextPage ?? 1; + var nextPageNumber = _outstandingAllTopicsScanContext?.NextPage ?? 1; var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for all topics - next available page is page {nextPageNumber}"; throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); } - // Get the list of topic names we need to query over, - // the current paging token if there is one & this isn't the first page, - // and the current shard to be paged over for the current topic - List topics; - string? paginationToken; - int currentShard; - if (pageNumber == 1) - { - topics = _topicNames.Keys.ToList(); - paginationToken = null; - currentShard = 0; - } - else - { - topics = _outstandingAllTopicsQueryContext!.RemainingTopics; - paginationToken = _outstandingAllTopicsQueryContext.LastEvaluatedKey; - currentShard = _outstandingAllTopicsQueryContext.ShardNumber; - } + // Get the current paging token if there is one & this isn't the first page + var paginationToken = pageNumber == 1 ? null : _outstandingAllTopicsScanContext?.LastEvaluatedKey; - // Iterate over topics and their associated shards until we reach the batch size - var results = new List(); - var currentTopicIndex = 0; - while (results.Count < pageSize && currentTopicIndex < topics.Count) + // Scan the outstanding (all topics) index to get outstanding messages up to the batch size + var allMessages = new List(); + do { - var remainingBatchSize = pageSize - results.Count; - var queryResult = await PageOutstandingMessagesToBatchSizeAsync( - topics[currentTopicIndex], - olderThan, - remainingBatchSize, - currentShard, - paginationToken, - cancellationToken); + var scanFilter = new ScanFilter(); + scanFilter.AddCondition("OutstandingCreatedTime", ScanOperator.LessThanOrEqual, + new List + { + new AttributeValue() + { + N = olderThan.Ticks.ToString() + } + }); + var scanConfig = new ScanOperationConfig + { + IndexName = _configuration.OutstandingAllTopicsIndexName, + ConsistentRead = false, + Filter = scanFilter, + Select = SelectValues.AllProjectedAttributes, + Limit = (int)(pageSize * 1.5), // Scan more than we need to account for filtering out recent messages + PaginationToken = paginationToken + }; + var scan = _context.FromScanAsync(scanConfig, _dynamoOverwriteTableConfig); - results.AddRange(queryResult.Messages); + allMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); - if (queryResult.QueryComplete) - { - currentTopicIndex++; - paginationToken = null; - currentShard = 0; - } - else - { - paginationToken = queryResult.PaginationToken; - currentShard = queryResult.ShardNumber; - } - } + paginationToken = scan.PaginationToken; + } while (paginationToken != null && allMessages.Count < pageSize); - // Store the progress for the "all topics" query if there are further pages - if (currentTopicIndex < topics.Count) + // If there are more results, store the context for retrieving the next page + if (paginationToken != null) { - var remainingTopics = topics.GetRange(currentTopicIndex, topics.Count - currentTopicIndex); - _outstandingAllTopicsQueryContext = new OutstandingAllTopicsQueryContext(pageNumber + 1, paginationToken!, currentShard, remainingTopics); + _outstandingAllTopicsScanContext = new AllTopicsScanContext(pageNumber + 1, paginationToken); } else { - _outstandingAllTopicsQueryContext = null; + _outstandingAllTopicsScanContext = null; } - return results.Select(msg => msg.ConvertToMessage()); + return allMessages + .OrderBy(m => m.OutstandingCreatedTime) + .Take(pageSize) + .Select(m => m.ConvertToMessage()); } private async Task> OutstandingMessagesForTopicAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, @@ -764,73 +742,67 @@ private async Task> DispatchedMessagesForTopicAsync( } private async Task> DispatchedMessagesForAllTopicsAsync( - TimeSpan dispatchedSince, + TimeSpan minimumAge, int pageSize, int pageNumber, CancellationToken cancellationToken) { - var sinceTime = _timeProvider.GetUtcNow() - dispatchedSince; + var dispatchedBefore = _timeProvider.GetUtcNow() - minimumAge; // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1 && _dispatchedAllTopicsQueryContext?.NextPage != pageNumber) + if (pageNumber != 1 && _dispatchedAllTopicsScanContext?.NextPage != pageNumber) { - var nextPageNumber = _dispatchedAllTopicsQueryContext?.NextPage ?? 1; + var nextPageNumber = _dispatchedAllTopicsScanContext?.NextPage ?? 1; var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for all topics - next available page is page {nextPageNumber}"; throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); } - // Get the list of topic names we need to query over, and the current paging token if there is one & this isn't the first page - List topics; - string? paginationToken; - if (pageNumber == 1) - { - topics = _topicNames.Keys.ToList(); - paginationToken = null; - } - else - { - topics = _dispatchedAllTopicsQueryContext!.RemainingTopics; - paginationToken = _dispatchedAllTopicsQueryContext.LastEvaluatedKey; - } + // Get the current paging token if there is one & this isn't the first page + var paginationToken = pageNumber == 1 ? null : _dispatchedAllTopicsScanContext?.LastEvaluatedKey; - // Iterate over topic until we reach the batch size - var results = new List(); - var currentTopicIndex = 0; - while (results.Count < pageSize && currentTopicIndex < topics.Count) + // Scan the delivered (all topics) index to get delivered messages up to the batch size + var allMessages = new List(); + do { - var remainingBatchSize = pageSize - results.Count; - var queryResult = await PageDispatchedMessagesToBatchSizeAsync( - topics[currentTopicIndex], - sinceTime, - remainingBatchSize, - paginationToken, - cancellationToken); - - results.AddRange(queryResult.Messages); - - if (queryResult.QueryComplete) - { - currentTopicIndex++; - paginationToken = null; - } - else + var scanFilter = new ScanFilter(); + scanFilter.AddCondition("DeliveryTime", ScanOperator.LessThanOrEqual, + new List + { + new AttributeValue() + { + N = dispatchedBefore.Ticks.ToString() + } + }); + var scanConfig = new ScanOperationConfig { - paginationToken = queryResult.PaginationToken; - } - } + IndexName = _configuration.DeliveredAllTopicsIndexName, + ConsistentRead = false, + Filter = scanFilter, + Select = SelectValues.AllProjectedAttributes, + Limit = (int)(pageSize * 1.5), // Scan more than we need to account for filtering out recent messages + PaginationToken = paginationToken + }; + var scan = _context.FromScanAsync(scanConfig, _dynamoOverwriteTableConfig); + + allMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); + + paginationToken = scan.PaginationToken; + } while (paginationToken != null && allMessages.Count < pageSize); - // Store the progress for the "all topics" query if there are further pages - if (currentTopicIndex < topics.Count) + // If there are more results, store the context for retrieving the next page + if (paginationToken != null) { - var outstandingTopics = topics.GetRange(currentTopicIndex, topics.Count - currentTopicIndex); - _dispatchedAllTopicsQueryContext = new DispatchedAllTopicsQueryContext(pageNumber + 1, paginationToken, outstandingTopics); + _dispatchedAllTopicsScanContext = new AllTopicsScanContext(pageNumber + 1, paginationToken); } else { - _dispatchedAllTopicsQueryContext = null; + _dispatchedAllTopicsScanContext = null; } - return results.Select(msg => msg.ConvertToMessage()); + return allMessages + .OrderBy(m => m.DeliveryTime) + .Take(pageSize) + .Select(m => m.ConvertToMessage()); } private async Task PageOutstandingMessagesToBatchSizeAsync( diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs b/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs index fa0ae24fba..751d2f19bf 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs @@ -31,7 +31,7 @@ public class MessageItem /// /// The content type as a . May be null. [DynamoDBProperty] - public string? ContentType { get; set; } + public string? ContentType { get; set; } /// /// The correlation id of the message @@ -58,6 +58,7 @@ public class MessageItem /// /// The outstanding creation time as a in ticks. May be null. [DynamoDBGlobalSecondaryIndexRangeKey(indexName: "Outstanding")] + [DynamoDBGlobalSecondaryIndexHashKey(indexName: "OutstandingAllTopics")] [DynamoDBProperty] public long? OutstandingCreatedTime { get; set; } @@ -91,6 +92,7 @@ public class MessageItem /// /// The delivery time as a in ticks. May be null. [DynamoDBGlobalSecondaryIndexRangeKey(indexName: "Delivered")] + [DynamoDBGlobalSecondaryIndexHashKey(indexName: "DeliveredAllTopics")] [DynamoDBProperty] public long? DeliveryTime { get; set; } @@ -115,6 +117,7 @@ public class MessageItem /// /// The message ID as a . May be null. [DynamoDBHashKey] + [DynamoDBGlobalSecondaryIndexRangeKey(indexNames: ["OutstandingAllTopics", "DeliveredAllTopics"])] [DynamoDBProperty] public string? MessageId { get; set; } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingAllTopicsQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingAllTopicsQueryContext.cs deleted file mode 100644 index a2bf4d6559..0000000000 --- a/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingAllTopicsQueryContext.cs +++ /dev/null @@ -1,44 +0,0 @@ -#region Licence - -/* The MIT License (MIT) -Copyright © 2024 Dominic Hickie - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. */ - -#endregion - -using System.Collections.Generic; - -namespace Paramore.Brighter.Outbox.DynamoDB; - -internal sealed class OutstandingAllTopicsQueryContext -{ - public int NextPage { get; private set; } - public string LastEvaluatedKey { get; private set; } - public int ShardNumber { get; private set; } - public List RemainingTopics { get; private set; } - - public OutstandingAllTopicsQueryContext(int nextPage, string lastEvaluatedKey, int shardNumber, List remainingTopics) - { - NextPage = nextPage; - LastEvaluatedKey = lastEvaluatedKey; - ShardNumber = shardNumber; - RemainingTopics = remainingTopics; - } -} From 9bdab4313fb90cbc0c95b4946f30cf2cb99a5941 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Fri, 12 Sep 2025 12:18:36 +0100 Subject: [PATCH 05/22] Add support for parallel scans of all topic indices This commit adds support for parallel scans when scanning for outstanding or delivered messages in the "all topics" indices. It also adds some semaphores to ensure that the paging state of either all topics or a specific topic is only operated on by one thread at a time. --- .../AllTopicsScanContext.cs | 35 ++- .../DynamoDbConfiguration.cs | 10 +- .../DynamoDbOutbox.cs | 294 +++++++++++------- .../OutstandingTopicQueryContext.cs | 33 -- ...icQueryContext.cs => TopicQueryContext.cs} | 25 +- 5 files changed, 250 insertions(+), 147 deletions(-) delete mode 100644 src/Paramore.Brighter.Outbox.DynamoDB/OutstandingTopicQueryContext.cs rename src/Paramore.Brighter.Outbox.DynamoDB/{DispatchedTopicQueryContext.cs => TopicQueryContext.cs} (63%) diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/AllTopicsScanContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB/AllTopicsScanContext.cs index 980b2fee3d..4001f8a4fb 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/AllTopicsScanContext.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/AllTopicsScanContext.cs @@ -23,10 +23,39 @@ THE SOFTWARE. */ #endregion +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + namespace Paramore.Brighter.Outbox.DynamoDB; -internal sealed class AllTopicsScanContext(int nextPage, string? lastEvaluatedKey) +internal sealed class AllTopicsScanContext(int numSegments) { - public int NextPage { get; private set; } = nextPage; - public string? LastEvaluatedKey { get; private set; } = lastEvaluatedKey; + public int NextPage { get; private set; } = 1; + + private string?[] _lastEvaluatedKeys = new string?[numSegments]; + + private SemaphoreSlim _scanLock = new(1, 1); + + public void SetPagingToken(int segmentNumber, string? lastEvaluatedKey) + { + _lastEvaluatedKeys[segmentNumber] = lastEvaluatedKey; + } + + public string? GetPagingToken(int segmentNumber) => _lastEvaluatedKeys[segmentNumber]; + + public void SetNextPage() + { + if (_lastEvaluatedKeys.Any(x => x != null)) + { + NextPage++; + } + else + { + NextPage = 1; + } + } + + public async Task Lock(CancellationToken cancellationToken) => await _scanLock.WaitAsync(cancellationToken); + public void Release() => _scanLock.Release(); } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs index 29077f876a..65f2ef03e5 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbConfiguration.cs @@ -44,14 +44,19 @@ public class DynamoDbConfiguration /// By default, messages will not expire /// public TimeSpan? TimeToLive { get; set; } - + + /// + /// The number of concurrent scans to use in a parallel scan when looking for outstanding messages + /// + public int ScanConcurrency { get; set; } + /// /// Create a DynamoDbConfiguration for Outbox support /// /// The name of the outbox table /// The timeout when talking to DynamoDb /// The number of shards; use more than one shard for active topics to avoid hotspots - public DynamoDbConfiguration(string? tableName = null, int timeout = 500, int numberOfShards = 3) + public DynamoDbConfiguration(string? tableName = null, int timeout = 500, int numberOfShards = 3, int scanConcurrency = 3) { TableName = tableName ?? "brighter_outbox"; OutstandingIndexName = "Outstanding"; @@ -60,6 +65,7 @@ public DynamoDbConfiguration(string? tableName = null, int timeout = 500, int nu DeliveredAllTopicsIndexName = "DeliveredAllTopics"; Timeout = timeout; NumberOfShards = numberOfShards; + ScanConcurrency = scanConcurrency; } } } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 2ebff4b197..3a3fa747be 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -48,11 +48,11 @@ public class DynamoDbOutbox : private readonly Random _random = new Random(); private readonly TimeProvider _timeProvider; - private readonly ConcurrentDictionary _outstandingTopicQueryContexts; - private readonly ConcurrentDictionary _dispatchedTopicQueryContexts; + private readonly ConcurrentDictionary _outstandingTopicQueryContexts; + private readonly ConcurrentDictionary _dispatchedTopicQueryContexts; - private AllTopicsScanContext? _outstandingAllTopicsScanContext; - private AllTopicsScanContext? _dispatchedAllTopicsScanContext; + private AllTopicsScanContext _outstandingAllTopicsScanContext; + private AllTopicsScanContext _dispatchedAllTopicsScanContext; private readonly InstrumentationOptions _instrumentationOptions; private const string DYNAMO_DB_NAME = "outbox"; @@ -93,8 +93,11 @@ public DynamoDbOutbox( throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); } - _outstandingTopicQueryContexts = new ConcurrentDictionary(); - _dispatchedTopicQueryContexts = new ConcurrentDictionary(); + _outstandingTopicQueryContexts = new ConcurrentDictionary(); + _dispatchedTopicQueryContexts = new ConcurrentDictionary(); + + _outstandingAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); + _dispatchedAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); _instrumentationOptions = instrumentationOptions; } @@ -116,8 +119,11 @@ public DynamoDbOutbox(DynamoDBContext context, DynamoDbConfiguration configurati throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); } - _outstandingTopicQueryContexts = new ConcurrentDictionary(); - _dispatchedTopicQueryContexts = new ConcurrentDictionary(); + _outstandingTopicQueryContexts = new ConcurrentDictionary(); + _dispatchedTopicQueryContexts = new ConcurrentDictionary(); + + _outstandingAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); + _dispatchedAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); } /// @@ -579,26 +585,67 @@ private async Task DeleteAsync( private async Task> OutstandingMessagesForAllTopicsAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, CancellationToken cancellationToken) { - var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; + // Only allow one outstanding messages scan at a time to ensure consistency of pagination tokens + await _outstandingAllTopicsScanContext.Lock(cancellationToken); + try + { + var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; + + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && _outstandingAllTopicsScanContext.NextPage != pageNumber) + { + var nextPageNumber = _outstandingAllTopicsScanContext.NextPage; + var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for all topics - next available page is page {nextPageNumber}"; + throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + } + + // Spin off requests to scan each segment + var tasks = new List>>(); + var segmentPageSize = pageSize / _configuration.ScanConcurrency; + for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) + { + tasks.Add(ScanOutstandingIndexSegment(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + } - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1 && _outstandingAllTopicsScanContext?.NextPage != pageNumber) + await Task.WhenAll(tasks); + + // Set the next page number based on the pagination tokens for the different segments + _outstandingAllTopicsScanContext.SetNextPage(); + + var allMessages = tasks.SelectMany(t => t.Result); + return allMessages + .OrderBy(m => m.OutstandingCreatedTime) + .Select(m => m.ConvertToMessage()); + } + finally { - var nextPageNumber = _outstandingAllTopicsScanContext?.NextPage ?? 1; - var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for all topics - next available page is page {nextPageNumber}"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + _outstandingAllTopicsScanContext.Release(); } + } - // Get the current paging token if there is one & this isn't the first page - var paginationToken = pageNumber == 1 ? null : _outstandingAllTopicsScanContext?.LastEvaluatedKey; - - // Scan the outstanding (all topics) index to get outstanding messages up to the batch size - var allMessages = new List(); + private async Task> ScanOutstandingIndexSegment(DateTimeOffset olderThan, + int pageSize, + int pageNumber, + int segmentNumber, + CancellationToken cancellationToken) + { + string? paginationToken = null; + if (pageNumber != 1 && paginationToken == null) + { + // It may be that this segment is done but other segments have more results + return new List(); + } + else if (pageNumber != 1) + { + paginationToken = _outstandingAllTopicsScanContext.GetPagingToken(segmentNumber); + } + + var segmentMessages = new List(); do { var scanFilter = new ScanFilter(); - scanFilter.AddCondition("OutstandingCreatedTime", ScanOperator.LessThanOrEqual, - new List + scanFilter.AddCondition("OutstandingCreatedTime", ScanOperator.LessThanOrEqual, + new List { new AttributeValue() { @@ -611,78 +658,76 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti ConsistentRead = false, Filter = scanFilter, Select = SelectValues.AllProjectedAttributes, - Limit = (int)(pageSize * 1.5), // Scan more than we need to account for filtering out recent messages - PaginationToken = paginationToken + Limit = pageSize - segmentMessages.Count, + PaginationToken = paginationToken, + Segment = segmentNumber, + TotalSegments = _configuration.ScanConcurrency }; var scan = _context.FromScanAsync(scanConfig, _dynamoOverwriteTableConfig); - allMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); + segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); paginationToken = scan.PaginationToken; - } while (paginationToken != null && allMessages.Count < pageSize); + } while (paginationToken != null && segmentMessages.Count < pageSize); // If there are more results, store the context for retrieving the next page if (paginationToken != null) { - _outstandingAllTopicsScanContext = new AllTopicsScanContext(pageNumber + 1, paginationToken); + _outstandingAllTopicsScanContext.SetPagingToken(segmentNumber, paginationToken); } else { - _outstandingAllTopicsScanContext = null; + _outstandingAllTopicsScanContext.SetPagingToken(segmentNumber, null); } - return allMessages - .OrderBy(m => m.OutstandingCreatedTime) - .Take(pageSize) - .Select(m => m.ConvertToMessage()); + return segmentMessages; } private async Task> OutstandingMessagesForTopicAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, string topicName, CancellationToken cancellationToken) { - var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; + var context = _outstandingTopicQueryContexts.GetOrAdd(topicName, new TopicQueryContext()); - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1) + await context.Lock(cancellationToken); + try { - if (!_outstandingTopicQueryContexts.TryGetValue(topicName, out OutstandingTopicQueryContext? context)) + var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; + + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && context.NextPage != pageNumber) { - var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for topic {topicName} - next available page is page 1"; + var nextPageNumber = context.NextPage; + var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for topic {topicName} - next available page is page {nextPageNumber}"; throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); } - if (context?.NextPage != pageNumber) + // Query as much as possible up to the max page (batch) size + string? paginationToken = null; + int initialShardNumber = 0; + if (pageNumber != 1) { - var nextPageNumber = _dispatchedTopicQueryContexts[topicName]?.NextPage ?? 1; - var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for topic {topicName} - next available page is page {nextPageNumber}"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + paginationToken = _outstandingTopicQueryContexts[topicName]!.LastEvaluatedKey; + initialShardNumber = _outstandingTopicQueryContexts[topicName]!.ShardNumber; } - } - // Query as much as possible up to the max page (batch) size - string? paginationToken = null; - int initialShardNumber = 0; - if (pageNumber != 1) - { - paginationToken = _outstandingTopicQueryContexts[topicName]!.LastEvaluatedKey; - initialShardNumber = _outstandingTopicQueryContexts[topicName]!.ShardNumber; - } + var queryResult = await PageOutstandingMessagesToBatchSizeAsync(topicName, olderThan, pageSize, initialShardNumber, paginationToken, cancellationToken); - var queryResult = await PageOutstandingMessagesToBatchSizeAsync(topicName, olderThan, pageSize, initialShardNumber, paginationToken, cancellationToken); + // Store the progress for this topic if there are further pages + if (!queryResult.QueryComplete) + { + context.SetPaginationState(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken); + } + else + { + context.Reset(); + } - // Store the progress for this topic if there are further pages - if (!queryResult.QueryComplete) - { - _outstandingTopicQueryContexts.AddOrUpdate(topicName, - new OutstandingTopicQueryContext(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken), - (_, _) => new OutstandingTopicQueryContext(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken)); + return queryResult.Messages.Select(msg => msg.ConvertToMessage()); } - else + finally { - _outstandingTopicQueryContexts.TryRemove(topicName, out _); + context.Release(); } - - return queryResult.Messages.Select(msg => msg.ConvertToMessage()); } private Task AddToTransactionWrite(MessageItem messageToStore, DynamoDbUnitOfWork dynamoDbUnitOfWork) @@ -703,42 +748,41 @@ private async Task> DispatchedMessagesForTopicAsync( string topicName, CancellationToken cancellationToken) { - var sinceTime = _timeProvider.GetUtcNow() - dispatchedSince; + var context = _dispatchedTopicQueryContexts.GetOrAdd(topicName, new TopicQueryContext()); - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1) + await context.Lock(cancellationToken); + try { - if (!_dispatchedTopicQueryContexts.TryGetValue(topicName, out var context)) - { - var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for topic {topicName} - next available page is page 1"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); - } + var sinceTime = _timeProvider.GetUtcNow() - dispatchedSince; - if (context?.NextPage != pageNumber) + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && pageNumber != context.NextPage) { - var nextPageNumber = _dispatchedTopicQueryContexts[topicName]?.NextPage ?? 1; + var nextPageNumber = context.NextPage; var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for topic {topicName} - next available page is page {nextPageNumber}"; throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); } - } - // Query as much as possible up to the max page (batch) size - var paginationToken = pageNumber == 1 ? null : _dispatchedTopicQueryContexts[topicName].LastEvaluatedKey; - var queryResult = await PageDispatchedMessagesToBatchSizeAsync(topicName, sinceTime, pageSize, paginationToken, cancellationToken); + // Query as much as possible up to the max page (batch) size + var paginationToken = pageNumber == 1 ? null : context.LastEvaluatedKey; + var queryResult = await PageDispatchedMessagesToBatchSizeAsync(topicName, sinceTime, pageSize, paginationToken, cancellationToken); - // Store the progress for this topic if there are further pages - if (!queryResult.QueryComplete) - { - _dispatchedTopicQueryContexts.AddOrUpdate(topicName, - new DispatchedTopicQueryContext(pageNumber + 1, queryResult.PaginationToken), - (_, _) => new DispatchedTopicQueryContext(pageNumber + 1, queryResult.PaginationToken)); + // Store the progress for this topic if there are further pages + if (!queryResult.QueryComplete) + { + context.SetPaginationState(pageNumber + 1, 0, queryResult.PaginationToken); + } + else + { + context.Reset(); + } + + return queryResult.Messages.Select(msg => msg.ConvertToMessage()); } - else + finally { - _dispatchedTopicQueryContexts.TryRemove(topicName, out _); + context.Release(); } - - return queryResult.Messages.Select(msg => msg.ConvertToMessage()); } private async Task> DispatchedMessagesForAllTopicsAsync( @@ -747,21 +791,62 @@ private async Task> DispatchedMessagesForAllTopicsAsync( int pageNumber, CancellationToken cancellationToken) { - var dispatchedBefore = _timeProvider.GetUtcNow() - minimumAge; + // Only allow one dispatched messages scan at a time to ensure consistency of pagination tokens + await _dispatchedAllTopicsScanContext.Lock(cancellationToken); + try + { + var dispatchedBefore = _timeProvider.GetUtcNow() - minimumAge; - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1 && _dispatchedAllTopicsScanContext?.NextPage != pageNumber) + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && _dispatchedAllTopicsScanContext.NextPage != pageNumber) + { + var nextPageNumber = _dispatchedAllTopicsScanContext.NextPage; + var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for all topics - next available page is page {nextPageNumber}"; + throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + } + + // Spin off requests to scan each segment + var tasks = new List>>(); + var segmentPageSize = pageSize / _configuration.ScanConcurrency; + for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) + { + tasks.Add(ScanDispatchedIndexSegment(dispatchedBefore, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + } + + await Task.WhenAll(tasks); + + // Set the next page number based on the pagination tokens for the different segments + _dispatchedAllTopicsScanContext.SetNextPage(); + + var allMessages = tasks.SelectMany(t => t.Result); + return allMessages + .OrderBy(m => m.DeliveryTime) + .Select(m => m.ConvertToMessage()); + } + finally { - var nextPageNumber = _dispatchedAllTopicsScanContext?.NextPage ?? 1; - var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for all topics - next available page is page {nextPageNumber}"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + _dispatchedAllTopicsScanContext.Release(); } + } - // Get the current paging token if there is one & this isn't the first page - var paginationToken = pageNumber == 1 ? null : _dispatchedAllTopicsScanContext?.LastEvaluatedKey; + private async Task> ScanDispatchedIndexSegment(DateTimeOffset dispatchedBefore, + int pageSize, + int pageNumber, + int segmentNumber, + CancellationToken cancellationToken) + { + string? paginationToken = null; + if (pageNumber != 1 && paginationToken == null) + { + // It may be that this segment is done but other segments have more results + return new List(); + } + else if (pageNumber != 1) + { + paginationToken = _dispatchedAllTopicsScanContext.GetPagingToken(segmentNumber); + } - // Scan the delivered (all topics) index to get delivered messages up to the batch size - var allMessages = new List(); + var segmentMessages = new List(); do { var scanFilter = new ScanFilter(); @@ -779,30 +864,29 @@ private async Task> DispatchedMessagesForAllTopicsAsync( ConsistentRead = false, Filter = scanFilter, Select = SelectValues.AllProjectedAttributes, - Limit = (int)(pageSize * 1.5), // Scan more than we need to account for filtering out recent messages - PaginationToken = paginationToken + Limit = pageSize - segmentMessages.Count, + PaginationToken = paginationToken, + Segment = segmentNumber, + TotalSegments = _configuration.ScanConcurrency }; var scan = _context.FromScanAsync(scanConfig, _dynamoOverwriteTableConfig); - - allMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); + + segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); paginationToken = scan.PaginationToken; - } while (paginationToken != null && allMessages.Count < pageSize); + } while (paginationToken != null && segmentMessages.Count < pageSize); // If there are more results, store the context for retrieving the next page if (paginationToken != null) { - _dispatchedAllTopicsScanContext = new AllTopicsScanContext(pageNumber + 1, paginationToken); + _dispatchedAllTopicsScanContext.SetPagingToken(segmentNumber, paginationToken); } else { - _dispatchedAllTopicsScanContext = null; + _dispatchedAllTopicsScanContext.SetPagingToken(segmentNumber, null); } - return allMessages - .OrderBy(m => m.DeliveryTime) - .Take(pageSize) - .Select(m => m.ConvertToMessage()); + return segmentMessages; } private async Task PageOutstandingMessagesToBatchSizeAsync( diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingTopicQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingTopicQueryContext.cs deleted file mode 100644 index e54e1f7bb7..0000000000 --- a/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingTopicQueryContext.cs +++ /dev/null @@ -1,33 +0,0 @@ -#region Licence - -/* The MIT License (MIT) -Copyright © 2024 Dominic Hickie - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. */ - -#endregion - -namespace Paramore.Brighter.Outbox.DynamoDB; - -internal sealed class OutstandingTopicQueryContext(int nextPage, int shardNumber, string? lastEvaluatedKey) -{ - public int NextPage { get; private set; } = nextPage; - public int ShardNumber { get; private set; } = shardNumber; - public string? LastEvaluatedKey { get; private set; } = lastEvaluatedKey; -} diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DispatchedTopicQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryContext.cs similarity index 63% rename from src/Paramore.Brighter.Outbox.DynamoDB/DispatchedTopicQueryContext.cs rename to src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryContext.cs index bd99e554e5..cc787dafe9 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DispatchedTopicQueryContext.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryContext.cs @@ -23,16 +23,33 @@ THE SOFTWARE. */ #endregion +using System.Threading; +using System.Threading.Tasks; + namespace Paramore.Brighter.Outbox.DynamoDB; -internal sealed class DispatchedTopicQueryContext +internal sealed class TopicQueryContext() { - public int NextPage { get; private set; } - public string LastEvaluatedKey { get; private set; } + public int NextPage { get; private set; } = 1; + public int ShardNumber { get; private set; } = 0; + public string? LastEvaluatedKey { get; private set; } = null; + + private SemaphoreSlim _queryLock = new(1, 1); - public DispatchedTopicQueryContext(int nextPage, string lastEvaluatedKey) + public void SetPaginationState(int nextPage, int shardNumber, string? lastEvaluatedKey) { NextPage = nextPage; + ShardNumber = shardNumber; LastEvaluatedKey = lastEvaluatedKey; } + + public void Reset() + { + NextPage = 1; + ShardNumber = 0; + LastEvaluatedKey = null; + } + + public async Task Lock(CancellationToken cancellationToken) => await _queryLock.WaitAsync(cancellationToken); + public void Release() => _queryLock.Release(); } From e8cad1ba4ab80fcfc7a047827a5d1c3572e26679 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Tue, 16 Sep 2025 15:43:24 +0100 Subject: [PATCH 06/22] Add sharding to delivered index This commit adds sharding to the delivered index, which is used when querying for delivered messages for a specific topic. It also takes advantage of this to refactor the code that iterates through shards in a sharded index to use common code for both the outstanding and delivered indices. --- .../DispatchedMessagesQueryResult.cs | 42 ---------- .../DynamoDbOutbox.cs | 84 +++++++++---------- ...agesQueryResult.cs => IndexQueryResult.cs} | 2 +- .../KeyTopicDeliveredTimeExpression.cs | 19 ++--- ...eyTopicOutstandingCreatedTimeExpression.cs | 17 ++-- .../MessageItem.cs | 3 +- .../TopicQueryKeyExpression.cs | 17 ++++ 7 files changed, 70 insertions(+), 114 deletions(-) delete mode 100644 src/Paramore.Brighter.Outbox.DynamoDB/DispatchedMessagesQueryResult.cs rename src/Paramore.Brighter.Outbox.DynamoDB/{OutstandingMessagesQueryResult.cs => IndexQueryResult.cs} (91%) create mode 100644 src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DispatchedMessagesQueryResult.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DispatchedMessagesQueryResult.cs deleted file mode 100644 index f28a8d105c..0000000000 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DispatchedMessagesQueryResult.cs +++ /dev/null @@ -1,42 +0,0 @@ -#region Licence - -/* The MIT License (MIT) -Copyright © 2024 Dominic Hickie - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. */ - -#endregion - -using System.Collections.Generic; - -namespace Paramore.Brighter.Outbox.DynamoDB; - -internal sealed class DispatchedMessagesQueryResult -{ - public IEnumerable Messages { get; private set; } - public string PaginationToken { get; private set; } - public bool QueryComplete { get; private set; } - - public DispatchedMessagesQueryResult(IEnumerable messages, string paginationToken, bool queryComplete) - { - Messages = messages; - PaginationToken = paginationToken; - QueryComplete = queryComplete; - } -} diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 3a3fa747be..0f0a9a4045 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -706,11 +706,18 @@ private async Task> OutstandingMessagesForTopicAsync(TimeSp int initialShardNumber = 0; if (pageNumber != 1) { - paginationToken = _outstandingTopicQueryContexts[topicName]!.LastEvaluatedKey; - initialShardNumber = _outstandingTopicQueryContexts[topicName]!.ShardNumber; + paginationToken = context.LastEvaluatedKey; + initialShardNumber = context.ShardNumber; } - var queryResult = await PageOutstandingMessagesToBatchSizeAsync(topicName, olderThan, pageSize, initialShardNumber, paginationToken, cancellationToken); + var queryResult = await PageIndexQueryToBatchSize(topicName, + olderThan, + pageSize, + initialShardNumber, + paginationToken, + _configuration.OutstandingIndexName, + new KeyTopicOutstandingCreatedTimeExpression(), + cancellationToken); // Store the progress for this topic if there are further pages if (!queryResult.QueryComplete) @@ -764,8 +771,22 @@ private async Task> DispatchedMessagesForTopicAsync( } // Query as much as possible up to the max page (batch) size - var paginationToken = pageNumber == 1 ? null : context.LastEvaluatedKey; - var queryResult = await PageDispatchedMessagesToBatchSizeAsync(topicName, sinceTime, pageSize, paginationToken, cancellationToken); + string? paginationToken = null; + int initialShardNumber = 0; + if (pageNumber != 1) + { + paginationToken = context.LastEvaluatedKey; + initialShardNumber = context.ShardNumber; + } + + var queryResult = await PageIndexQueryToBatchSize(topicName, + sinceTime, + pageSize, + initialShardNumber, + paginationToken, + _configuration.DeliveredIndexName, + new KeyTopicDeliveredTimeExpression(), + cancellationToken); // Store the progress for this topic if there are further pages if (!queryResult.QueryComplete) @@ -889,31 +910,33 @@ private async Task> ScanDispatchedIndexSegment(DateTimeOffset return segmentMessages; } - private async Task PageOutstandingMessagesToBatchSizeAsync( - string topicName, - DateTimeOffset olderThan, + private async Task PageIndexQueryToBatchSize( + string topicName, + DateTimeOffset sinceTime, int batchSize, - int initialShardNumber, + int initialShardNumber, string? initialPaginationToken, + string indexName, + TopicQueryKeyExpression keyExpression, CancellationToken cancellationToken) { var numShards = _configuration.NumberOfShards <= 1 ? 1 : _configuration.NumberOfShards; - var results = new List(); + var results = new List(); var paginationToken = initialPaginationToken; var isDone = false; var shard = initialShardNumber; + while (shard < numShards && results.Count < batchSize) { do { var queryConfig = new QueryOperationConfig { - IndexName = _configuration.OutstandingIndexName, - KeyExpression = new KeyTopicOutstandingCreatedTimeExpression().Generate(topicName, olderThan, shard), + IndexName = indexName, + KeyExpression = keyExpression.Generate(topicName, sinceTime, shard), Limit = batchSize - results.Count, - PaginationToken = paginationToken, - ConsistentRead = false + PaginationToken = paginationToken }; var asyncSearch = _context.FromQueryAsync(queryConfig, _dynamoOverwriteTableConfig); @@ -947,40 +970,9 @@ private async Task PageOutstandingMessagesToBatc queryComplete = false; } - return new OutstandingMessagesQueryResult(results, nextShardNumber, paginationToken, queryComplete); + return new IndexQueryResult(results, nextShardNumber, paginationToken, queryComplete); } - private async Task PageDispatchedMessagesToBatchSizeAsync( - string topicName, - DateTimeOffset sinceTime, - int batchSize, - string? initialPaginationToken, - CancellationToken cancellationToken) - { - var results = new List(); - var keyExpression = new KeyTopicDeliveredTimeExpression().Generate(topicName, sinceTime); - var paginationToken = initialPaginationToken; - var isDone = false; - do - { - var queryConfig = new QueryOperationConfig - { - IndexName = _configuration.DeliveredIndexName, - KeyExpression = keyExpression, - Limit = batchSize - results.Count, - PaginationToken = paginationToken - }; - - var asyncSearch = _context.FromQueryAsync(queryConfig, _dynamoOverwriteTableConfig); - results.AddRange(await asyncSearch.GetNextSetAsync(cancellationToken)); - - paginationToken = asyncSearch.PaginationToken; - isDone = asyncSearch.IsDone; - } while (results.Count < batchSize && !isDone); - - return new DispatchedMessagesQueryResult(results, paginationToken, isDone); - } - private async Task WriteMessageToOutbox(CancellationToken cancellationToken, MessageItem messageToStore) { await _context.SaveAsync( diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingMessagesQueryResult.cs b/src/Paramore.Brighter.Outbox.DynamoDB/IndexQueryResult.cs similarity index 91% rename from src/Paramore.Brighter.Outbox.DynamoDB/OutstandingMessagesQueryResult.cs rename to src/Paramore.Brighter.Outbox.DynamoDB/IndexQueryResult.cs index 0c4b381e70..327402e131 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/OutstandingMessagesQueryResult.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/IndexQueryResult.cs @@ -27,7 +27,7 @@ THE SOFTWARE. */ namespace Paramore.Brighter.Outbox.DynamoDB; -internal sealed class OutstandingMessagesQueryResult(IEnumerable messages, int shardNumber, string? paginationToken, bool queryComplete) +internal sealed class IndexQueryResult(IEnumerable messages, int shardNumber, string? paginationToken, bool queryComplete) { public IEnumerable Messages { get; private set; } = messages; public int ShardNumber { get; private set; } = shardNumber; diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicDeliveredTimeExpression.cs b/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicDeliveredTimeExpression.cs index 9e1a84cd9a..bd7fd3178e 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicDeliveredTimeExpression.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicDeliveredTimeExpression.cs @@ -4,27 +4,22 @@ namespace Paramore.Brighter.Outbox.DynamoDB { - internal sealed class KeyTopicDeliveredTimeExpression + internal sealed class KeyTopicDeliveredTimeExpression : TopicQueryKeyExpression { - private readonly Expression _expression = new() + public KeyTopicDeliveredTimeExpression() { - ExpressionStatement = "Topic = :v_Topic and DeliveryTime < :v_SinceTime" - }; - - public override string ToString() - { - return _expression.ExpressionStatement; + Expression.ExpressionStatement = "TopicShard = :v_TopicShard and DeliveryTime < :v_SinceTime"; } - public Expression Generate(string topicName, DateTimeOffset sinceTime) + public override Expression Generate(string topicName, DateTimeOffset sinceTime, int shard) { - _expression.ExpressionAttributeValues = new Dictionary(capacity: 2) + Expression.ExpressionAttributeValues = new Dictionary(capacity: 2) { - { ":v_Topic", topicName }, + { ":v_TopicShard", $"{topicName}_{shard}" }, { ":v_SinceTime", sinceTime.Ticks } }; - return _expression; + return Expression; } } } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicOutstandingCreatedTimeExpression.cs b/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicOutstandingCreatedTimeExpression.cs index 5f227ff73f..087bdad948 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicOutstandingCreatedTimeExpression.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/KeyTopicOutstandingCreatedTimeExpression.cs @@ -4,27 +4,22 @@ namespace Paramore.Brighter.Outbox.DynamoDB { - internal sealed class KeyTopicOutstandingCreatedTimeExpression + internal sealed class KeyTopicOutstandingCreatedTimeExpression : TopicQueryKeyExpression { - private readonly Expression _expression = new() + public KeyTopicOutstandingCreatedTimeExpression() { - ExpressionStatement = "TopicShard = :v_TopicShard and OutstandingCreatedTime < :v_OutstandingCreatedTime" - }; - - public override string ToString() - { - return _expression.ExpressionStatement; + Expression.ExpressionStatement = "TopicShard = :v_TopicShard and OutstandingCreatedTime < :v_OutstandingCreatedTime"; } - public Expression Generate(string topicName, DateTimeOffset createdTime, int shard) + public override Expression Generate(string topicName, DateTimeOffset createdTime, int shard) { - _expression.ExpressionAttributeValues = new Dictionary(capacity: 2) + Expression.ExpressionAttributeValues = new Dictionary(capacity: 2) { { ":v_TopicShard", $"{topicName}_{shard}" }, { ":v_OutstandingCreatedTime", createdTime.Ticks } }; - return _expression; + return Expression; } } } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs b/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs index 751d2f19bf..99e99c71f6 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/MessageItem.cs @@ -163,7 +163,6 @@ public class MessageItem /// The Topic the message was published to /// /// The topic as a . May be null. - [DynamoDBGlobalSecondaryIndexHashKey("Delivered")] [DynamoDBProperty] public string? Topic { get; set; } @@ -171,7 +170,7 @@ public class MessageItem /// The Topic suffixed with the shard number /// /// The topic shard as a . May be null. - [DynamoDBGlobalSecondaryIndexHashKey("Outstanding")] + [DynamoDBGlobalSecondaryIndexHashKey(["Outstanding", "Delivered"])] [DynamoDBProperty] public string? TopicShard { get; set; } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs b/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs new file mode 100644 index 0000000000..45fbdef092 --- /dev/null +++ b/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs @@ -0,0 +1,17 @@ +using System; +using Amazon.DynamoDBv2.DocumentModel; + +namespace Paramore.Brighter.Outbox.DynamoDB +{ + internal abstract class TopicQueryKeyExpression + { + internal Expression Expression { get; } = new(); + + public override string ToString() + { + return Expression.ExpressionStatement; + } + + public abstract Expression Generate(string topicName, DateTimeOffset sinceTime, int shard); + } +} From 52558507246891f5f8d4ea80e3c70bc78000b853 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Wed, 17 Sep 2025 11:29:12 +0100 Subject: [PATCH 07/22] Add BatchGet method to outboxes This commit adds both sync and async methods for getting a batch of messages from the outbox, and adds an implementation using BatchGet for the Dynamo DB outbox. It also updates some method signatures for other outbox implementations to align to the new interface, and adds implementations where missing. Finally, it copies all the optimisations made so far over to the AWS SDK v4 version of the Dynamo outbox. --- ...QueryResult.cs => AllTopicsScanContext.cs} | 37 +- .../DispatchedAllTopicsQueryContext.cs | 35 - .../DynamoDbConfiguration.cs | 115 +- .../DynamoDbOutbox.cs | 1649 +++++++++-------- ...agesQueryResult.cs => IndexQueryResult.cs} | 2 +- .../KeyTopicDeliveredTimeExpression.cs | 34 +- ...eyTopicOutstandingCreatedTimeExpression.cs | 34 +- .../OutstandingTopicQueryContext.cs | 33 - ...csQueryContext.cs => TopicQueryContext.cs} | 31 +- ...yContext.cs => TopicQueryKeyExpression.cs} | 18 +- .../DynamoDbOutbox.cs | 60 +- .../TopicQueryKeyExpression.cs | 45 +- .../MongoDbOutbox.cs | 12 +- .../ControlBus/ControlBusReceiverBuilder.cs | 5 + src/Paramore.Brighter/IAmAnOutboxAsync.cs | 19 +- src/Paramore.Brighter/IAmAnOutboxSync.cs | 16 +- src/Paramore.Brighter/InMemoryOutbox.cs | 45 + .../RelationDatabaseOutbox.cs | 10 +- .../TestDoubles/SpyOutbox.cs | 10 + 19 files changed, 1192 insertions(+), 1018 deletions(-) rename src/Paramore.Brighter.Outbox.DynamoDB.V4/{DispatchedMessagesQueryResult.cs => AllTopicsScanContext.cs} (57%) delete mode 100644 src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedAllTopicsQueryContext.cs rename src/Paramore.Brighter.Outbox.DynamoDB.V4/{OutstandingMessagesQueryResult.cs => IndexQueryResult.cs} (91%) delete mode 100644 src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingTopicQueryContext.cs rename src/Paramore.Brighter.Outbox.DynamoDB.V4/{OutstandingAllTopicsQueryContext.cs => TopicQueryContext.cs} (65%) rename src/Paramore.Brighter.Outbox.DynamoDB.V4/{DispatchedTopicQueryContext.cs => TopicQueryKeyExpression.cs} (74%) diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedMessagesQueryResult.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/AllTopicsScanContext.cs similarity index 57% rename from src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedMessagesQueryResult.cs rename to src/Paramore.Brighter.Outbox.DynamoDB.V4/AllTopicsScanContext.cs index eee91d281d..6c4da75a80 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedMessagesQueryResult.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/AllTopicsScanContext.cs @@ -23,20 +23,39 @@ THE SOFTWARE. */ #endregion -using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; namespace Paramore.Brighter.Outbox.DynamoDB.V4; -internal sealed class DispatchedMessagesQueryResult +internal sealed class AllTopicsScanContext(int numSegments) { - public IEnumerable Messages { get; private set; } - public string PaginationToken { get; private set; } - public bool QueryComplete { get; private set; } + public int NextPage { get; private set; } = 1; - public DispatchedMessagesQueryResult(IEnumerable messages, string paginationToken, bool queryComplete) + private string?[] _lastEvaluatedKeys = new string?[numSegments]; + + private SemaphoreSlim _scanLock = new(1, 1); + + public void SetPagingToken(int segmentNumber, string? lastEvaluatedKey) { - Messages = messages; - PaginationToken = paginationToken; - QueryComplete = queryComplete; + _lastEvaluatedKeys[segmentNumber] = lastEvaluatedKey; } + + public string? GetPagingToken(int segmentNumber) => _lastEvaluatedKeys[segmentNumber]; + + public void SetNextPage() + { + if (_lastEvaluatedKeys.Any(x => x != null)) + { + NextPage++; + } + else + { + NextPage = 1; + } + } + + public async Task Lock(CancellationToken cancellationToken) => await _scanLock.WaitAsync(cancellationToken); + public void Release() => _scanLock.Release(); } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedAllTopicsQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedAllTopicsQueryContext.cs deleted file mode 100644 index 5d7ddf7295..0000000000 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedAllTopicsQueryContext.cs +++ /dev/null @@ -1,35 +0,0 @@ -#region Licence - -/* The MIT License (MIT) -Copyright © 2024 Dominic Hickie - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. */ - -#endregion - -using System.Collections.Generic; - -namespace Paramore.Brighter.Outbox.DynamoDB.V4; - -internal sealed class DispatchedAllTopicsQueryContext(int nextPage, string? lastEvaluatedKey, List remainingTopics) -{ - public int NextPage { get; private set; } = nextPage; - public string? LastEvaluatedKey { get; private set; } = lastEvaluatedKey; - public List RemainingTopics { get; private set; } = remainingTopics; -} diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbConfiguration.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbConfiguration.cs index a9d2b74b12..7acbb0ab7d 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbConfiguration.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbConfiguration.cs @@ -1,52 +1,71 @@ using System; -namespace Paramore.Brighter.Outbox.DynamoDB.V4; - -public class DynamoDbConfiguration -{ - /// - /// The table that forms the Outbox - /// - public string TableName { get; set; } - - /// - /// The name of local delivered status index - /// - public string DeliveredIndexName { get; } - - /// - /// The name of the global secondary outstanding index - /// - public string OutstandingIndexName { get; set; } - - /// - /// Timeout in milliseconds - /// - public int Timeout { get; } - - /// - /// Number of shards to use for the outstanding index. Maximum of 20 - /// - public int NumberOfShards { get; } - - /// - /// Optional time to live for the messages in the outbox - /// By default, messages will not expire - /// - public TimeSpan? TimeToLive { get; set; } - - /// - /// Create a DynamoDbConfiguration for Outbox support - /// - /// The name of the outbox table - /// The timeout when talking to DynamoDb - /// The number of shards; use more than one shard for active topics to avoid hotspots - public DynamoDbConfiguration(string? tableName = null, int timeout = 500, int numberOfShards = 3) +namespace Paramore.Brighter.Outbox.DynamoDB.V4 +{ + public class DynamoDbConfiguration { - TableName = tableName ?? "brighter_outbox"; - OutstandingIndexName = "Outstanding"; - DeliveredIndexName = "Delivered"; - Timeout = timeout; - NumberOfShards = numberOfShards; + /// + /// The table that forms the Outbox + /// + public string TableName { get; set; } + + /// + /// The name of the global secondary delivered index indexed by topic + /// + public string DeliveredIndexName { get; set; } + + /// + /// The name of the global secondary delivered index convering all topics + /// + public string DeliveredAllTopicsIndexName { get; set; } + + /// + /// The name of the global secondary outstanding index indexed by topic + /// + public string OutstandingIndexName { get; set; } + + /// + /// The name of the global secondary outstanding index convering all topics + /// + public string OutstandingAllTopicsIndexName { get; set; } + + /// + /// Timeout in milliseconds + /// + public int Timeout { get; } + + /// + /// Number of shards to use for the outstanding index. Maximum of 20 + /// + public int NumberOfShards { get; } + + /// + /// Optional time to live for the messages in the outbox + /// By default, messages will not expire + /// + public TimeSpan? TimeToLive { get; set; } + + /// + /// The number of concurrent scans to use in a parallel scan when looking for outstanding messages + /// + public int ScanConcurrency { get; set; } + + /// + /// Create a DynamoDbConfiguration for Outbox support + /// + /// The name of the outbox table + /// The timeout when talking to DynamoDb + /// The number of shards; use more than one shard for active topics to avoid hotspots + public DynamoDbConfiguration(string? tableName = null, int timeout = 500, int numberOfShards = 3, int scanConcurrency = 3) + { + TableName = tableName ?? "brighter_outbox"; + OutstandingIndexName = "Outstanding"; + OutstandingAllTopicsIndexName = "OutstandingAllTopics"; + DeliveredIndexName = "Delivered"; + DeliveredAllTopicsIndexName = "DeliveredAllTopics"; + Timeout = timeout; + NumberOfShards = numberOfShards; + ScanConcurrency = scanConcurrency; + } } -} \ No newline at end of file +} diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index 7df3fa3f2a..e0b5e05b96 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -33,945 +33,1020 @@ THE SOFTWARE. */ using Amazon.DynamoDBv2.DataModel; using Amazon.DynamoDBv2.DocumentModel; using Amazon.DynamoDBv2.Model; -using Paramore.Brighter.DynamoDb; using Paramore.Brighter.DynamoDb.V4; using Paramore.Brighter.Observability; -namespace Paramore.Brighter.Outbox.DynamoDB.V4; - -public class DynamoDbOutbox : - IAmAnOutboxSync, - IAmAnOutboxAsync +namespace Paramore.Brighter.Outbox.DynamoDB.V4 { - private readonly DynamoDbConfiguration _configuration; - private readonly DynamoDBContext _context; - private readonly LoadConfig _loadConfig; - private readonly SaveConfig _saveConfig; - private readonly DeleteConfig _deleteConfig; - private readonly FromQueryConfig _fromQueryConfig; - private readonly ToDocumentConfig _toDocumentConfig; - private readonly Random _random = new Random(); - private readonly TimeProvider _timeProvider; - - private readonly ConcurrentDictionary _outstandingTopicQueryContexts; - private readonly ConcurrentDictionary _dispatchedTopicQueryContexts; - - private readonly ConcurrentDictionary _topicNames; + public class DynamoDbOutbox : + IAmAnOutboxSync, + IAmAnOutboxAsync + { + private readonly DynamoDbConfiguration _configuration; + private readonly DynamoDBContext _context; + private readonly LoadConfig _loadConfig; + private readonly BatchGetConfig _batchGetConfig; + private readonly SaveConfig _saveConfig; + private readonly DeleteConfig _deleteConfig; + private readonly FromQueryConfig _fromQueryConfig; + private readonly FromScanConfig _fromScanConfig; + private readonly ToDocumentConfig _toDocumentConfig; + private readonly Random _random = new Random(); + private readonly TimeProvider _timeProvider; + + private readonly ConcurrentDictionary _outstandingTopicQueryContexts; + private readonly ConcurrentDictionary _dispatchedTopicQueryContexts; + + private AllTopicsScanContext _outstandingAllTopicsScanContext; + private AllTopicsScanContext _dispatchedAllTopicsScanContext; + + private readonly InstrumentationOptions _instrumentationOptions; + private const string DYNAMO_DB_NAME = "outbox"; + + public bool ContinueOnCapturedContext { get; set; } + + /// + /// The Tracer that we want to use to capture telemetry + /// We inject this so that we can use the same tracer as the calling application + /// You do not need to set this property as we will set it when setting up the External Service Bus + /// + public IAmABrighterTracer? Tracer { private get; set; } + + /// + /// Initialises a new instance of the class. + /// + /// The DynamoDBContext + /// The DynamoDB Operation Configuration + /// Provides a timer that can be overwritten in teests; on null uses system timer + /// + public DynamoDbOutbox( + IAmazonDynamoDB client, + DynamoDbConfiguration configuration, + TimeProvider? timeProvider = null, + InstrumentationOptions instrumentationOptions = InstrumentationOptions.All) + { + _configuration = configuration; + _context = new DynamoDBContextBuilder().WithDynamoDBClient(() => client).Build(); + _timeProvider = timeProvider ?? TimeProvider.System; + _loadConfig = new LoadConfig + { + OverrideTableName = _configuration.TableName, + ConsistentRead = true + }; - private OutstandingAllTopicsQueryContext? _outstandingAllTopicsQueryContext; - private DispatchedAllTopicsQueryContext? _dispatchedAllTopicsQueryContext; + _saveConfig = new SaveConfig { OverrideTableName = _configuration.TableName }; + _deleteConfig = new DeleteConfig { OverrideTableName = _configuration.TableName, }; + _fromQueryConfig = new FromQueryConfig { OverrideTableName = _configuration.TableName }; + _fromScanConfig = new FromScanConfig { OverrideTableName = _configuration.TableName }; + _toDocumentConfig = new ToDocumentConfig { OverrideTableName = _configuration.TableName, }; + _batchGetConfig = new BatchGetConfig { OverrideTableName = _configuration.TableName }; - private readonly InstrumentationOptions _instrumentationOptions; - private const string DYNAMO_DB_NAME = "outbox"; + if (_configuration.NumberOfShards > 20) + { + throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); + } - public bool ContinueOnCapturedContext { get; set; } - - /// - /// The Tracer that we want to use to capture telemetry - /// We inject this so that we can use the same tracer as the calling application - /// You do not need to set this property as we will set it when setting up the External Service Bus - /// - public IAmABrighterTracer? Tracer { private get; set; } - - /// - /// Initialises a new instance of the class. - /// - /// The DynamoDBContext - /// The DynamoDB Operation Configuration - /// Provides a timer that can be overwritten in teests; on null uses system timer - /// - public DynamoDbOutbox( - IAmazonDynamoDB client, - DynamoDbConfiguration configuration, - TimeProvider? timeProvider = null, - InstrumentationOptions instrumentationOptions = InstrumentationOptions.All) - { - _configuration = configuration; - _context = new DynamoDBContextBuilder().WithDynamoDBClient(() => client).Build(); - _timeProvider = timeProvider ?? TimeProvider.System; - new DynamoDBOperationConfig - { - OverrideTableName = _configuration.TableName, - ConsistentRead = true - }; + _outstandingTopicQueryContexts = new ConcurrentDictionary(); + _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - _loadConfig = new LoadConfig - { - OverrideTableName = _configuration.TableName, - ConsistentRead = true - }; - - _saveConfig = new SaveConfig { OverrideTableName = _configuration.TableName }; - _deleteConfig = new DeleteConfig { OverrideTableName = _configuration.TableName, }; - _fromQueryConfig = new FromQueryConfig { OverrideTableName = _configuration.TableName }; - _toDocumentConfig = new ToDocumentConfig { OverrideTableName = _configuration.TableName, }; + _outstandingAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); + _dispatchedAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); + _instrumentationOptions = instrumentationOptions; + } - if (_configuration.NumberOfShards > 20) + /// + /// Initialises a new instance of the class. + /// + /// An existing Dynamo Db Context + /// The Configuration from the context - the config is internal, so we can't grab the settings from it. + public DynamoDbOutbox(DynamoDBContext context, DynamoDbConfiguration configuration, TimeProvider timeProvider) { - throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); - } + _context = context; + _configuration = configuration; + _timeProvider = timeProvider; + _loadConfig = new LoadConfig + { + OverrideTableName = _configuration.TableName, + ConsistentRead = true + }; - _outstandingTopicQueryContexts = new ConcurrentDictionary(); - _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - _topicNames = new ConcurrentDictionary(); + _saveConfig = new SaveConfig { OverrideTableName = _configuration.TableName }; + _deleteConfig = new DeleteConfig { OverrideTableName = _configuration.TableName, }; + _fromQueryConfig = new FromQueryConfig { OverrideTableName = _configuration.TableName }; + _fromScanConfig = new FromScanConfig { OverrideTableName = _configuration.TableName }; + _toDocumentConfig = new ToDocumentConfig { OverrideTableName = _configuration.TableName, }; + _batchGetConfig = new BatchGetConfig { OverrideTableName = _configuration.TableName }; - _instrumentationOptions = instrumentationOptions; - } + if (_configuration.NumberOfShards > 20) + { + throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); + } - /// - /// Initialises a new instance of the class. - /// - /// An existing Dynamo Db Context - /// The Configuration from the context - the config is internal, so we can't grab the settings from it. - public DynamoDbOutbox(DynamoDBContext context, DynamoDbConfiguration configuration, TimeProvider timeProvider) - { - _context = context; - _configuration = configuration; - _timeProvider = timeProvider; - new DynamoDBOperationConfig { OverrideTableName = _configuration.TableName }; - _loadConfig = new LoadConfig { OverrideTableName = _configuration.TableName }; - _saveConfig = new SaveConfig { OverrideTableName = _configuration.TableName }; - _deleteConfig = new DeleteConfig { OverrideTableName = _configuration.TableName, }; - _fromQueryConfig = new FromQueryConfig { OverrideTableName = _configuration.TableName }; - _toDocumentConfig = new ToDocumentConfig { OverrideTableName = _configuration.TableName, }; + _outstandingTopicQueryContexts = new ConcurrentDictionary(); + _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - - if (_configuration.NumberOfShards > 20) - { - throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); + _outstandingAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); + _dispatchedAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); } - _outstandingTopicQueryContexts = new ConcurrentDictionary(); - _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - _topicNames = new ConcurrentDictionary(); - } - - /// - /// - /// Adds a message to the Outbox - /// Sync over async - /// - /// The message to be stored - /// What is the context of this request; used to provide Span information to the call - /// Timeout in milliseconds; -1 for default timeout - /// Should we participate in a transaction - public void Add( - Message message, - RequestContext? requestContext, - int outBoxTimeout = -1, - IAmABoxTransactionProvider? transactionProvider = null - ) - { - AddAsync(message, requestContext, outBoxTimeout, transactionProvider).ConfigureAwait(ContinueOnCapturedContext).GetAwaiter().GetResult(); - } - - /// - /// Adds messages to the Outbox - /// - /// The messages to be stored - /// What is the context for this request; used to access the Span - /// Timeout in milliseconds; -1 for default timeout - /// Should we participate in a transaction - public void Add( - IEnumerable messages, - RequestContext? requestContext, - int outBoxTimeout = -1, - IAmABoxTransactionProvider? transactionProvider = null - ) - { - foreach (var message in messages) + /// + /// + /// Adds a message to the Outbox + /// Sync over async + /// + /// The message to be stored + /// What is the context of this request; used to provide Span information to the call + /// Timeout in milliseconds; -1 for default timeout + /// Should we participate in a transaction + public void Add( + Message message, + RequestContext? requestContext, + int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null + ) { - Add(message, requestContext, outBoxTimeout, transactionProvider); + AddAsync(message, requestContext, outBoxTimeout, transactionProvider).ConfigureAwait(ContinueOnCapturedContext).GetAwaiter().GetResult(); } - } - /// - /// Adds a message to the Outbox - /// - /// The message to be stored - /// What is the context for this request; used to access the Span - /// Timeout in milliseconds; -1 for default timeout - /// Should we participate in a transaction - /// Allows the sender to cancel the request pipeline. Optional - public async Task AddAsync( - Message message, - RequestContext? requestContext, - int outBoxTimeout = -1, - IAmABoxTransactionProvider? transactionProvider = null, - CancellationToken cancellationToken = default) - { - var dbAttributes = new Dictionary() + /// + /// Adds messages to the Outbox + /// + /// The messages to be stored + /// What is the context for this request; used to access the Span + /// Timeout in milliseconds; -1 for default timeout + /// Should we participate in a transaction + public void Add( + IEnumerable messages, + RequestContext? requestContext, + int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null + ) { - {"db.operation.parameter.message.id", message.Id} - }; - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Add, _configuration.TableName, dbAttributes: dbAttributes), - requestContext?.Span, - options: _instrumentationOptions); + foreach (var message in messages) + { + Add(message, requestContext, outBoxTimeout, transactionProvider); + } + } - try + /// + /// Adds a message to the Outbox + /// + /// The message to be stored + /// What is the context for this request; used to access the Span + /// Timeout in milliseconds; -1 for default timeout + /// Should we participate in a transaction + /// Allows the sender to cancel the request pipeline. Optional + public async Task AddAsync( + Message message, + RequestContext? requestContext, + int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null, + CancellationToken cancellationToken = default) { - var shard = GetShardNumber(); - var expiresAt = GetExpirationTime(); - var messageToStore = new MessageItem(message, shard, expiresAt); - - // Store the name of the topic as a key in a concurrent dictionary to ensure uniqueness & thread safety - _topicNames.TryAdd(message.Header.Topic, 0); + var dbAttributes = new Dictionary() + { + {"db.operation.parameter.message.id", message.Id} + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Add, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); - if (transactionProvider != null) + try { - await AddToTransactionWrite(messageToStore, (DynamoDbUnitOfWork)transactionProvider); + var shard = GetShardNumber(); + var expiresAt = GetExpirationTime(); + var messageToStore = new MessageItem(message, shard, expiresAt); + + if (transactionProvider != null) + { + await AddToTransactionWrite(messageToStore, (DynamoDbUnitOfWork)transactionProvider); + } + else + { + await WriteMessageToOutbox(cancellationToken, messageToStore); + } } - else + finally { - await WriteMessageToOutbox(cancellationToken, messageToStore); - } + Tracer?.EndSpan(span); + } } - finally - { - Tracer?.EndSpan(span); - } - } - /// - /// Adds messages to the Outbox - /// - /// The messages to be stored - /// What is the context for this request; used to access the Span - /// Timeout in milliseconds; -1 for default timeout - /// - /// Allows the sender to cancel the request pipeline. Optional - public async Task AddAsync( - IEnumerable messages, - RequestContext? requestContext, - int outBoxTimeout = -1, - IAmABoxTransactionProvider? transactionProvider = null, - CancellationToken cancellationToken = default) - { - foreach (var message in messages) + /// + /// Adds messages to the Outbox + /// + /// The messages to be stored + /// What is the context for this request; used to access the Span + /// Timeout in milliseconds; -1 for default timeout + /// + /// Allows the sender to cancel the request pipeline. Optional + public async Task AddAsync( + IEnumerable messages, + RequestContext? requestContext, + int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null, + CancellationToken cancellationToken = default) { - await AddAsync(message, requestContext, outBoxTimeout, transactionProvider, cancellationToken); + foreach (var message in messages) + { + await AddAsync(message, requestContext, outBoxTimeout, transactionProvider, cancellationToken); + } } - } - /// - /// Delete messages from the Outbox - /// Sync over async - /// - /// The messages to delete - /// What is the context for this request; used to access the Span - /// Additional parameters required to search if needed - public void Delete(Id[] messageIds, RequestContext? requestContext, Dictionary? args = null) - { - DeleteAsync(messageIds, requestContext).GetAwaiter().GetResult(); - } - - /// - /// Delete messages from the Outbox - /// - /// The messages to delete - /// What is the context for this request; used to access the Span - /// Additional parameters required to search if needed - /// Should the operation be cancelled - public async Task DeleteAsync( - Id[] messageIds, - RequestContext? requestContext, - Dictionary? args = null, - CancellationToken cancellationToken = default) - { - foreach (var messageId in messageIds) + /// + /// Delete messages from the Outbox + /// Sync over async + /// + /// The messages to delete + /// What is the context for this request; used to access the Span + /// Additional parameters required to search if needed + public void Delete(Id[] messageIds, RequestContext? requestContext, Dictionary? args = null) { - await DeleteAsync(messageId, requestContext, args, cancellationToken); + DeleteAsync(messageIds, requestContext).GetAwaiter().GetResult(); } - } - - /// - /// Returns messages that have been successfully dispatched. Eventually consistent. - /// Sync over async - /// - /// How long ago was the message dispatched? - /// What is the context for this request; used to access the Span - /// How many messages returned at once? - /// Which page of the dispatched messages to return? - /// - /// Used to pass through the topic we are searching for messages in. Use Key: "Topic" - /// A list of dispatched messages - public IEnumerable DispatchedMessages( - TimeSpan dispatchedSince, - RequestContext requestContext, - int pageSize = 100, - int pageNumber = 1, - int outboxTimeout = -1, - Dictionary? args = null) - { - return DispatchedMessagesAsync(dispatchedSince, requestContext, pageSize, pageNumber, outboxTimeout, args, CancellationToken.None) - .GetAwaiter().GetResult(); - } - - /// - /// Returns messages that have been successfully dispatched. Eventually consistent. - /// - /// How long ago was the message dispatched? - /// What is the context for this request; used to access the Span - /// How many messages returned at once? - /// Which page of the dispatched messages to return? - /// - /// Used to pass through the topic we are searching for messages in. Use Key: "Topic" - /// Cancel the running operation - /// A list of dispatched messages - /// - public async Task> DispatchedMessagesAsync( - TimeSpan dispatchedSince, - RequestContext requestContext, - int pageSize = 100, - int pageNumber = 1, - int outboxTimeout = -1, - Dictionary? args = null, - CancellationToken cancellationToken = default) - { - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.DispatchedMessages, _configuration.TableName), - requestContext?.Span, - options: _instrumentationOptions); - try + /// + /// Delete messages from the Outbox + /// + /// The messages to delete + /// What is the context for this request; used to access the Span + /// Additional parameters required to search if needed + /// Should the operation be cancelled + public async Task DeleteAsync( + Id[] messageIds, + RequestContext? requestContext, + Dictionary? args = null, + CancellationToken cancellationToken = default) { - IEnumerable result; - if (args == null || !args.TryGetValue("Topic", out var topicArg)) - { - result = await DispatchedMessagesForAllTopicsAsync(dispatchedSince, pageSize, pageNumber, cancellationToken); - } - else + foreach (var messageId in messageIds) { - result = await DispatchedMessagesForTopicAsync(dispatchedSince, pageSize, pageNumber, (string)topicArg, cancellationToken); + await DeleteAsync(messageId, requestContext, args, cancellationToken); } - - span?.AddTag("db.response.returned_rows", result.Count()); - return result; } - finally + + /// + /// Returns messages that have been successfully dispatched. Eventually consistent. + /// Sync over async + /// + /// How long ago was the message dispatched? + /// What is the context for this request; used to access the Span + /// How many messages returned at once? + /// Which page of the dispatched messages to return? + /// + /// Used to pass through the topic we are searching for messages in. Use Key: "Topic" + /// A list of dispatched messages + public IEnumerable DispatchedMessages( + TimeSpan dispatchedSince, + RequestContext requestContext, + int pageSize = 100, + int pageNumber = 1, + int outboxTimeout = -1, + Dictionary? args = null) { - Tracer?.EndSpan(span); + return DispatchedMessagesAsync(dispatchedSince, requestContext, pageSize, pageNumber, outboxTimeout, args, CancellationToken.None) + .GetAwaiter().GetResult(); } - } - - /// - /// Returns messages that have been successfully dispatched. Eventually consistent. - /// - /// How many hours back to look - /// What is the context for this request; used to access the Span - /// The number of results to return. Only returns this number of results - /// How to cancel - /// - public async Task> DispatchedMessagesAsync( - TimeSpan dispatchedSince, - RequestContext requestContext, - int pageSize = 100, - CancellationToken cancellationToken = default - ) - { - return await DispatchedMessagesAsync(dispatchedSince, requestContext, pageSize: pageSize, pageNumber: 1, outboxTimeout: -1, args: null, cancellationToken: cancellationToken); - } - - /// - /// Finds a message with the specified identifier. - /// Sync over async - /// - /// The identifier. - /// What is the context for this request; used to access the Span - /// Timeout in milliseconds; -1 for default timeout - /// - /// - public Message Get(Id messageId, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) - { - return GetAsync(messageId, requestContext, outBoxTimeout, args).ConfigureAwait(ContinueOnCapturedContext).GetAwaiter().GetResult(); - } - /// - /// Finds a message with the specified identifier. - /// - /// The identifier. - /// What is the context for this request; used to access the Span - /// Timeout in milliseconds; -1 for default timeout - /// For outboxes that require additional parameters such as topic, provide an optional arg - /// - /// - public async Task GetAsync( - Id messageId, - RequestContext requestContext, - int outBoxTimeout = -1, - Dictionary? args = null, - CancellationToken cancellationToken = default) - { - var dbAttributes = new Dictionary() + /// + /// Returns messages that have been successfully dispatched. Eventually consistent. + /// + /// How long ago was the message dispatched? + /// What is the context for this request; used to access the Span + /// How many messages returned at once? + /// Which page of the dispatched messages to return? + /// + /// Used to pass through the topic we are searching for messages in. Use Key: "Topic" + /// Cancel the running operation + /// A list of dispatched messages + /// + public async Task> DispatchedMessagesAsync( + TimeSpan dispatchedSince, + RequestContext requestContext, + int pageSize = 100, + int pageNumber = 1, + int outboxTimeout = -1, + Dictionary? args = null, + CancellationToken cancellationToken = default) { - {"db.operation.parameter.message.id", messageId} - }; - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Get, _configuration.TableName, dbAttributes: dbAttributes), - requestContext?.Span, - options: _instrumentationOptions); + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.DispatchedMessages, _configuration.TableName), + requestContext?.Span, + options: _instrumentationOptions); - try - { - var messageItem = await _context.LoadAsync(messageId.Value, _loadConfig, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - return messageItem?.ConvertToMessage() ?? new Message(); + try + { + IEnumerable result; + if (args == null || !args.TryGetValue("Topic", out var topicArg)) + { + result = await DispatchedMessagesForAllTopicsAsync(dispatchedSince, pageSize, pageNumber, cancellationToken); + } + else + { + result = await DispatchedMessagesForTopicAsync(dispatchedSince, pageSize, pageNumber, (string)topicArg, cancellationToken); + } + + span?.AddTag("db.response.returned_rows", result.Count()); + return result; + } + finally + { + Tracer?.EndSpan(span); + } } - finally + + /// + /// Returns messages that have been successfully dispatched. Eventually consistent. + /// + /// How many hours back to look + /// What is the context for this request; used to access the Span + /// The number of results to return. Only returns this number of results + /// How to cancel + /// + public async Task> DispatchedMessagesAsync( + TimeSpan dispatchedSince, + RequestContext requestContext, + int pageSize = 100, + CancellationToken cancellationToken = default + ) { - Tracer?.EndSpan(span); + return await DispatchedMessagesAsync(dispatchedSince, requestContext, pageSize: pageSize, pageNumber: 1, outboxTimeout: -1, args: null, cancellationToken: cancellationToken); } - } - /// - /// Update a message to show it is dispatched - /// - /// The id of the message to update - /// What is the context for this request; used to access the Span - /// When was the message dispatched, defaults to UTC now - /// - /// Allows the sender to cancel the request pipeline. Optional - public async Task MarkDispatchedAsync( - Id id, - RequestContext requestContext, - DateTimeOffset? dispatchedAt = null, - Dictionary? args = null, - CancellationToken cancellationToken = default) - { - var dbAttributes = new Dictionary() + /// + public Message Get(Id messageId, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) { - {"db.operation.parameter.message.id", id} - }; - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.MarkDispatched, _configuration.TableName, dbAttributes: dbAttributes), - requestContext?.Span, - options: _instrumentationOptions); + return GetAsync(messageId, requestContext, outBoxTimeout, args).ConfigureAwait(ContinueOnCapturedContext).GetAwaiter().GetResult(); + } - try + /// + public async Task GetAsync( + Id messageId, + RequestContext requestContext, + int outBoxTimeout = -1, + Dictionary? args = null, + CancellationToken cancellationToken = default) { - var message = await _context.LoadAsync(id.Value, _loadConfig, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - MarkMessageDispatched(dispatchedAt ?? _timeProvider.GetUtcNow(), message); + var dbAttributes = new Dictionary() + { + {"db.operation.parameter.message.id", messageId} + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Get, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); - await _context.SaveAsync( - message, - _saveConfig, - cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - } - finally - { - Tracer?.EndSpan(span); + try + { + var messageItem = await _context.LoadAsync(messageId.Value, _loadConfig, cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + return messageItem?.ConvertToMessage() ?? new Message(); + } + finally + { + Tracer?.EndSpan(span); + } } - } - /// - /// Marks a set of messages as dispatched in the Outbox - /// - /// The ides of the messages to mark as dispatched - /// What is the context for this request; used to access the Span - /// What time were the messages dispatched at? Defaults to Utc.Now - /// What is the topic of the message - /// Cancel an ongoing operation - public async Task MarkDispatchedAsync( - IEnumerable ids, - RequestContext requestContext, - DateTimeOffset? dispatchedAt = null, - Dictionary? args = null, - CancellationToken cancellationToken = default) - { - foreach(var messageId in ids) + /// + public IEnumerable Get(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) { - await MarkDispatchedAsync(messageId, requestContext, dispatchedAt, args, cancellationToken); + return GetAsync(messageIds, requestContext, outBoxTimeout, args).ConfigureAwait(ContinueOnCapturedContext).GetAwaiter().GetResult(); } - } - /// - /// Update a message to show it is dispatched - /// - /// The id of the message to update - /// What is the context for this request; used to access the Span - /// When was the message dispatched, defaults to UTC now - /// - public void MarkDispatched(Id id, RequestContext requestContext, DateTimeOffset? dispatchedAt = null, Dictionary? args = null) - { - MarkDispatchedAsync(id, requestContext, dispatchedAt, args) - .ConfigureAwait(ContinueOnCapturedContext) - .GetAwaiter() - .GetResult(); - } - - private static void MarkMessageDispatched(DateTimeOffset dispatchedAt, MessageItem message) - { - message.DeliveryTime = dispatchedAt.Ticks; - message.DeliveredAt = dispatchedAt.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); - - // Set the outstanding created time to null to remove the attribute - // from the item in dynamo - message.OutstandingCreatedTime = null; - } - - /// - /// Returns messages that have yet to be dispatched - /// Sync over async - /// - /// How long ago as the message sent? - /// What is the context for this request; used to access the Span - /// How many messages to return at once? - /// Which page number of messages - /// Collection of tripped topics - /// - /// A list of messages that are outstanding for dispatch - public IEnumerable OutstandingMessages( - TimeSpan dispatchedSince, - RequestContext? requestContext, - int pageSize = 100, - int pageNumber = 1, - IEnumerable? trippedTopics = null, - Dictionary? args = null) - { - return OutstandingMessagesAsync(dispatchedSince, requestContext, pageSize, pageNumber, trippedTopics, args) - .GetAwaiter() - .GetResult(); - } - - /// - /// Returns messages that have yet to be dispatched - /// - /// How long ago as the message sent? - /// - /// How many messages to return at once? - /// Which page number of messages - /// Collection of tripped topics - /// - /// Async Cancellation Token - /// A list of messages that are outstanding for dispatch - public async Task> OutstandingMessagesAsync( - TimeSpan dispatchedSince, - RequestContext? requestContext, - int pageSize = 100, - int pageNumber = 1, - IEnumerable? trippedTopics = null, - Dictionary? args = null, - CancellationToken cancellationToken = default) - { - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.OutStandingMessages, _configuration.TableName), - requestContext?.Span, - options: _instrumentationOptions); - - try + /// + public async Task> GetAsync( + IEnumerable messageIds, + RequestContext requestContext, + int outBoxTimeout = -1, + Dictionary? args = null, + CancellationToken cancellationToken = default) { - IEnumerable result; - if (args == null || !args.TryGetValue("Topic", out var topicArg)) + var dbAttributes = new Dictionary() + { + {"db.operation.parameter.message.ids", string.Join(",", messageIds.Select(x => x.ToString()))} + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Get, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); + + try { - result = await OutstandingMessagesForAllTopicsAsync(dispatchedSince, pageSize, pageNumber, cancellationToken); + var batchGet = _context.CreateBatchGet(_batchGetConfig); + foreach (var id in messageIds) + { + batchGet.AddKey(id.Value); + } + + await batchGet.ExecuteAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + return batchGet.Results.Select(x => x.ConvertToMessage()); } - else + finally { - result = await OutstandingMessagesForTopicAsync(dispatchedSince, pageSize, pageNumber, (string)topicArg, cancellationToken); + Tracer?.EndSpan(span); } - - span?.AddTag("db.response.returned_rows", result.Count()); - return result; - } - finally - { - Tracer?.EndSpan(span); } - } - private async Task DeleteAsync( - Id messageId, - RequestContext? requestContext, - Dictionary? args, - CancellationToken cancellationToken) - { - var dbAttributes = new Dictionary() + /// + /// Update a message to show it is dispatched + /// + /// The id of the message to update + /// What is the context for this request; used to access the Span + /// When was the message dispatched, defaults to UTC now + /// + /// Allows the sender to cancel the request pipeline. Optional + public async Task MarkDispatchedAsync( + Id id, + RequestContext requestContext, + DateTimeOffset? dispatchedAt = null, + Dictionary? args = null, + CancellationToken cancellationToken = default) { - {"db.operation.parameter.message.id", messageId} - }; - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Delete, _configuration.TableName, dbAttributes: dbAttributes), - requestContext?.Span, - options: _instrumentationOptions); + var dbAttributes = new Dictionary() + { + {"db.operation.parameter.message.id", id} + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.MarkDispatched, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); - try - { - await _context.DeleteAsync(messageId.Value, _deleteConfig, cancellationToken); + try + { + var message = await _context.LoadAsync(id.Value, _loadConfig, cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + MarkMessageDispatched(dispatchedAt ?? _timeProvider.GetUtcNow(), message); + + await _context.SaveAsync( + message, + _saveConfig, + cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + } + finally + { + Tracer?.EndSpan(span); + } } - finally + + /// + /// Marks a set of messages as dispatched in the Outbox + /// + /// The ides of the messages to mark as dispatched + /// What is the context for this request; used to access the Span + /// What time were the messages dispatched at? Defaults to Utc.Now + /// What is the topic of the message + /// Cancel an ongoing operation + public async Task MarkDispatchedAsync( + IEnumerable ids, + RequestContext requestContext, + DateTimeOffset? dispatchedAt = null, + Dictionary? args = null, + CancellationToken cancellationToken = default) { - Tracer?.EndSpan(span); + foreach(var messageId in ids) + { + await MarkDispatchedAsync(messageId, requestContext, dispatchedAt, args, cancellationToken); + } } - } - private async Task> OutstandingMessagesForAllTopicsAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, CancellationToken cancellationToken) - { - var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; - - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1 && _outstandingAllTopicsQueryContext?.NextPage != pageNumber) + /// + /// Update a message to show it is dispatched + /// + /// The id of the message to update + /// What is the context for this request; used to access the Span + /// When was the message dispatched, defaults to UTC now + /// + public void MarkDispatched(Id id, RequestContext requestContext, DateTimeOffset? dispatchedAt = null, Dictionary? args = null) { - var nextPageNumber = _outstandingAllTopicsQueryContext?.NextPage ?? 1; - var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for all topics - next available page is page {nextPageNumber}"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + MarkDispatchedAsync(id, requestContext, dispatchedAt, args) + .ConfigureAwait(ContinueOnCapturedContext) + .GetAwaiter() + .GetResult(); } - // Get the list of topic names we need to query over, - // the current paging token if there is one & this isn't the first page, - // and the current shard to be paged over for the current topic - List topics; - string? paginationToken; - int currentShard; - if (pageNumber == 1) + private static void MarkMessageDispatched(DateTimeOffset dispatchedAt, MessageItem message) { - topics = _topicNames.Keys.ToList(); - paginationToken = null; - currentShard = 0; + message.DeliveryTime = dispatchedAt.Ticks; + message.DeliveredAt = dispatchedAt.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + + // Set the outstanding created time to null to remove the attribute + // from the item in dynamo + message.OutstandingCreatedTime = null; } - else + + /// + /// Returns messages that have yet to be dispatched + /// Sync over async + /// + /// How long ago as the message sent? + /// What is the context for this request; used to access the Span + /// How many messages to return at once? + /// Which page number of messages + /// Collection of tripped topics + /// + /// A list of messages that are outstanding for dispatch + public IEnumerable OutstandingMessages( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int pageSize = 100, + int pageNumber = 1, + IEnumerable? trippedTopics = null, + Dictionary? args = null) { - topics = _outstandingAllTopicsQueryContext!.RemainingTopics; - paginationToken = _outstandingAllTopicsQueryContext.LastEvaluatedKey; - currentShard = _outstandingAllTopicsQueryContext.ShardNumber; + return OutstandingMessagesAsync(dispatchedSince, requestContext, pageSize, pageNumber, args: args) + .GetAwaiter() + .GetResult(); } - // Iterate over topics and their associated shards until we reach the batch size - var results = new List(); - var currentTopicIndex = 0; - while (results.Count < pageSize && currentTopicIndex < topics.Count) + /// + /// Returns messages that have yet to be dispatched + /// + /// How long ago as the message sent? + /// + /// How many messages to return at once? + /// Which page number of messages + /// Collection of tripped topics + /// + /// Async Cancellation Token + /// A list of messages that are outstanding for dispatch + public async Task> OutstandingMessagesAsync( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int pageSize = 100, + int pageNumber = 1, + IEnumerable? trippedTopics = null, + Dictionary? args = null, + CancellationToken cancellationToken = default) { - var remainingBatchSize = pageSize - results.Count; - var queryResult = await PageOutstandingMessagesToBatchSizeAsync( - topics[currentTopicIndex], - olderThan, - remainingBatchSize, - currentShard, - paginationToken, - cancellationToken); - - results.AddRange(queryResult.Messages); + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.OutStandingMessages, _configuration.TableName), + requestContext?.Span, + options: _instrumentationOptions); - if (queryResult.QueryComplete) + try { - currentTopicIndex++; - paginationToken = null; - currentShard = 0; + IEnumerable result; + if (args == null || !args.TryGetValue("Topic", out var topicArg)) + { + result = await OutstandingMessagesForAllTopicsAsync(dispatchedSince, pageSize, pageNumber, cancellationToken); + } + else + { + result = await OutstandingMessagesForTopicAsync(dispatchedSince, pageSize, pageNumber, (string)topicArg, cancellationToken); + } + + span?.AddTag("db.response.returned_rows", result.Count()); + return result; } - else + finally { - paginationToken = queryResult.PaginationToken; - currentShard = queryResult.ShardNumber; + Tracer?.EndSpan(span); } } - // Store the progress for the "all topics" query if there are further pages - if (currentTopicIndex < topics.Count) + private async Task DeleteAsync( + Id messageId, + RequestContext? requestContext, + Dictionary? args, + CancellationToken cancellationToken) { - var remainingTopics = topics.GetRange(currentTopicIndex, topics.Count - currentTopicIndex); - _outstandingAllTopicsQueryContext = new OutstandingAllTopicsQueryContext(pageNumber + 1, paginationToken!, currentShard, remainingTopics); - } - else - { - _outstandingAllTopicsQueryContext = null; - } - - return results.Select(msg => msg.ConvertToMessage()); - } - - private async Task> OutstandingMessagesForTopicAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, - string topicName, CancellationToken cancellationToken) - { - var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; + var dbAttributes = new Dictionary() + { + {"db.operation.parameter.message.id", messageId} + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Delete, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1) - { - if (!_outstandingTopicQueryContexts.TryGetValue(topicName, out OutstandingTopicQueryContext? context)) + try { - var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for topic {topicName} - next available page is page 1"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + await _context.DeleteAsync(messageId.Value, _deleteConfig, cancellationToken); } - - if (context?.NextPage != pageNumber) + finally { - var nextPageNumber = _dispatchedTopicQueryContexts[topicName]?.NextPage ?? 1; - var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for topic {topicName} - next available page is page {nextPageNumber}"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + Tracer?.EndSpan(span); } } - // Query as much as possible up to the max page (batch) size - string? paginationToken = null; - int initialShardNumber = 0; - if (pageNumber != 1) + private async Task> OutstandingMessagesForAllTopicsAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, CancellationToken cancellationToken) { - paginationToken = _outstandingTopicQueryContexts[topicName]!.LastEvaluatedKey; - initialShardNumber = _outstandingTopicQueryContexts[topicName]!.ShardNumber; - } - - var queryResult = await PageOutstandingMessagesToBatchSizeAsync(topicName, olderThan, pageSize, initialShardNumber, paginationToken, cancellationToken); + // Only allow one outstanding messages scan at a time to ensure consistency of pagination tokens + await _outstandingAllTopicsScanContext.Lock(cancellationToken); + try + { + var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; - // Store the progress for this topic if there are further pages - if (!queryResult.QueryComplete) - { - _outstandingTopicQueryContexts.AddOrUpdate(topicName, - new OutstandingTopicQueryContext(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken), - (_, _) => new OutstandingTopicQueryContext(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken)); - } - else - { - _outstandingTopicQueryContexts.TryRemove(topicName, out _); - } + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && _outstandingAllTopicsScanContext.NextPage != pageNumber) + { + var nextPageNumber = _outstandingAllTopicsScanContext.NextPage; + var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for all topics - next available page is page {nextPageNumber}"; + throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + } + + // Spin off requests to scan each segment + var tasks = new List>>(); + var segmentPageSize = pageSize / _configuration.ScanConcurrency; + for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) + { + tasks.Add(ScanOutstandingIndexSegment(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + } - return queryResult.Messages.Select(msg => msg.ConvertToMessage()); - } + await Task.WhenAll(tasks); - private Task AddToTransactionWrite(MessageItem messageToStore, DynamoDbUnitOfWork dynamoDbUnitOfWork) - { - var tcs = new TaskCompletionSource(); - var attributes = _context.ToDocument(messageToStore, _toDocumentConfig).ToAttributeMap(); - - var transaction = dynamoDbUnitOfWork.GetTransaction(); - transaction.TransactItems.Add(new TransactWriteItem{Put = new Put{TableName = _configuration.TableName, Item = attributes}}); - tcs.SetResult(transaction); - return tcs.Task; - } + // Set the next page number based on the pagination tokens for the different segments + _outstandingAllTopicsScanContext.SetNextPage(); - private async Task> DispatchedMessagesForTopicAsync( - TimeSpan dispatchedSince, - int pageSize, - int pageNumber, - string topicName, - CancellationToken cancellationToken) - { - var sinceTime = _timeProvider.GetUtcNow() - dispatchedSince; + var allMessages = tasks.SelectMany(t => t.Result); + return allMessages + .OrderBy(m => m.OutstandingCreatedTime) + .Select(m => m.ConvertToMessage()); + } + finally + { + _outstandingAllTopicsScanContext.Release(); + } + } - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1) + private async Task> ScanOutstandingIndexSegment(DateTimeOffset olderThan, + int pageSize, + int pageNumber, + int segmentNumber, + CancellationToken cancellationToken) { - if (!_dispatchedTopicQueryContexts.TryGetValue(topicName, out var context)) + string? paginationToken = null; + if (pageNumber != 1 && paginationToken == null) { - var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for topic {topicName} - next available page is page 1"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + // It may be that this segment is done but other segments have more results + return new List(); } - - if (context?.NextPage != pageNumber) + else if (pageNumber != 1) { - var nextPageNumber = _dispatchedTopicQueryContexts[topicName]?.NextPage ?? 1; - var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for topic {topicName} - next available page is page {nextPageNumber}"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + paginationToken = _outstandingAllTopicsScanContext.GetPagingToken(segmentNumber); } - } + + var segmentMessages = new List(); + do + { + var scanFilter = new ScanFilter(); + scanFilter.AddCondition("OutstandingCreatedTime", ScanOperator.LessThanOrEqual, + new List + { + new AttributeValue() + { + N = olderThan.Ticks.ToString() + } + }); + var scanConfig = new ScanOperationConfig + { + IndexName = _configuration.OutstandingAllTopicsIndexName, + ConsistentRead = false, + Filter = scanFilter, + Select = SelectValues.AllProjectedAttributes, + Limit = pageSize - segmentMessages.Count, + PaginationToken = paginationToken, + Segment = segmentNumber, + TotalSegments = _configuration.ScanConcurrency + }; + var scan = _context.FromScanAsync(scanConfig, _fromScanConfig); - // Query as much as possible up to the max page (batch) size - var paginationToken = pageNumber == 1 ? null : _dispatchedTopicQueryContexts[topicName].LastEvaluatedKey; - var queryResult = await PageDispatchedMessagesToBatchSizeAsync(topicName, sinceTime, pageSize, paginationToken, cancellationToken); + segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); - // Store the progress for this topic if there are further pages - if (!queryResult.QueryComplete) - { - _dispatchedTopicQueryContexts.AddOrUpdate(topicName, - new DispatchedTopicQueryContext(pageNumber + 1, queryResult.PaginationToken), - (_, _) => new DispatchedTopicQueryContext(pageNumber + 1, queryResult.PaginationToken)); + paginationToken = scan.PaginationToken; + } while (paginationToken != null && segmentMessages.Count < pageSize); + + // If there are more results, store the context for retrieving the next page + if (paginationToken != null) + { + _outstandingAllTopicsScanContext.SetPagingToken(segmentNumber, paginationToken); + } + else + { + _outstandingAllTopicsScanContext.SetPagingToken(segmentNumber, null); + } + + return segmentMessages; } - else + + private async Task> OutstandingMessagesForTopicAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, + string topicName, CancellationToken cancellationToken) { - _dispatchedTopicQueryContexts.TryRemove(topicName, out _); - } + var context = _outstandingTopicQueryContexts.GetOrAdd(topicName, new TopicQueryContext()); - return queryResult.Messages.Select(msg => msg.ConvertToMessage()); - } + await context.Lock(cancellationToken); + try + { + var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; - private async Task> DispatchedMessagesForAllTopicsAsync( - TimeSpan dispatchedSince, - int pageSize, - int pageNumber, - CancellationToken cancellationToken) - { - var sinceTime = _timeProvider.GetUtcNow() - dispatchedSince; + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && context.NextPage != pageNumber) + { + var nextPageNumber = context.NextPage; + var errorMessage = $"Unable to query page {pageNumber} of outstanding messages for topic {topicName} - next available page is page {nextPageNumber}"; + throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + } + + // Query as much as possible up to the max page (batch) size + string? paginationToken = null; + int initialShardNumber = 0; + if (pageNumber != 1) + { + paginationToken = context.LastEvaluatedKey; + initialShardNumber = context.ShardNumber; + } + + var queryResult = await PageIndexQueryToBatchSize(topicName, + olderThan, + pageSize, + initialShardNumber, + paginationToken, + _configuration.OutstandingIndexName, + new KeyTopicOutstandingCreatedTimeExpression(), + cancellationToken); + + // Store the progress for this topic if there are further pages + if (!queryResult.QueryComplete) + { + context.SetPaginationState(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken); + } + else + { + context.Reset(); + } - // Validate that this is a query for a page we can actually retrieve - if (pageNumber != 1 && _dispatchedAllTopicsQueryContext?.NextPage != pageNumber) - { - var nextPageNumber = _dispatchedAllTopicsQueryContext?.NextPage ?? 1; - var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for all topics - next available page is page {nextPageNumber}"; - throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + return queryResult.Messages.Select(msg => msg.ConvertToMessage()); + } + finally + { + context.Release(); + } } - // Get the list of topic names we need to query over, and the current paging token if there is one & this isn't the first page - List topics; - string? paginationToken; - if (pageNumber == 1) + private Task AddToTransactionWrite(MessageItem messageToStore, DynamoDbUnitOfWork dynamoDbUnitOfWork) { - topics = _topicNames.Keys.ToList(); - paginationToken = null; - } - else - { - topics = _dispatchedAllTopicsQueryContext!.RemainingTopics; - paginationToken = _dispatchedAllTopicsQueryContext.LastEvaluatedKey; + var tcs = new TaskCompletionSource(); + var attributes = _context.ToDocument(messageToStore, _toDocumentConfig).ToAttributeMap(); + + var transaction = dynamoDbUnitOfWork.GetTransaction(); + transaction.TransactItems.Add(new TransactWriteItem{Put = new Put{TableName = _configuration.TableName, Item = attributes}}); + tcs.SetResult(transaction); + return tcs.Task; } - // Iterate over topic until we reach the batch size - var results = new List(); - var currentTopicIndex = 0; - while (results.Count < pageSize && currentTopicIndex < topics.Count) + private async Task> DispatchedMessagesForTopicAsync( + TimeSpan dispatchedSince, + int pageSize, + int pageNumber, + string topicName, + CancellationToken cancellationToken) { - var remainingBatchSize = pageSize - results.Count; - var queryResult = await PageDispatchedMessagesToBatchSizeAsync( - topics[currentTopicIndex], - sinceTime, - remainingBatchSize, - paginationToken, - cancellationToken); + var context = _dispatchedTopicQueryContexts.GetOrAdd(topicName, new TopicQueryContext()); - results.AddRange(queryResult.Messages); - - if (queryResult.QueryComplete) + await context.Lock(cancellationToken); + try { - currentTopicIndex++; - paginationToken = null; + var sinceTime = _timeProvider.GetUtcNow() - dispatchedSince; + + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && pageNumber != context.NextPage) + { + var nextPageNumber = context.NextPage; + var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for topic {topicName} - next available page is page {nextPageNumber}"; + throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + } + + // Query as much as possible up to the max page (batch) size + string? paginationToken = null; + int initialShardNumber = 0; + if (pageNumber != 1) + { + paginationToken = context.LastEvaluatedKey; + initialShardNumber = context.ShardNumber; + } + + var queryResult = await PageIndexQueryToBatchSize(topicName, + sinceTime, + pageSize, + initialShardNumber, + paginationToken, + _configuration.DeliveredIndexName, + new KeyTopicDeliveredTimeExpression(), + cancellationToken); + + // Store the progress for this topic if there are further pages + if (!queryResult.QueryComplete) + { + context.SetPaginationState(pageNumber + 1, 0, queryResult.PaginationToken); + } + else + { + context.Reset(); + } + + return queryResult.Messages.Select(msg => msg.ConvertToMessage()); } - else + finally { - paginationToken = queryResult.PaginationToken; + context.Release(); } } - // Store the progress for the "all topics" query if there are further pages - if (currentTopicIndex < topics.Count) - { - var outstandingTopics = topics.GetRange(currentTopicIndex, topics.Count - currentTopicIndex); - _dispatchedAllTopicsQueryContext = new DispatchedAllTopicsQueryContext(pageNumber + 1, paginationToken, outstandingTopics); - } - else + private async Task> DispatchedMessagesForAllTopicsAsync( + TimeSpan minimumAge, + int pageSize, + int pageNumber, + CancellationToken cancellationToken) { - _dispatchedAllTopicsQueryContext = null; - } + // Only allow one dispatched messages scan at a time to ensure consistency of pagination tokens + await _dispatchedAllTopicsScanContext.Lock(cancellationToken); + try + { + var dispatchedBefore = _timeProvider.GetUtcNow() - minimumAge; - return results.Select(msg => msg.ConvertToMessage()); - } + // Validate that this is a query for a page we can actually retrieve + if (pageNumber != 1 && _dispatchedAllTopicsScanContext.NextPage != pageNumber) + { + var nextPageNumber = _dispatchedAllTopicsScanContext.NextPage; + var errorMessage = $"Unable to query page {pageNumber} of dispatched messages for all topics - next available page is page {nextPageNumber}"; + throw new ArgumentOutOfRangeException(nameof(pageNumber), errorMessage); + } + + // Spin off requests to scan each segment + var tasks = new List>>(); + var segmentPageSize = pageSize / _configuration.ScanConcurrency; + for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) + { + tasks.Add(ScanDispatchedIndexSegment(dispatchedBefore, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + } - private async Task PageOutstandingMessagesToBatchSizeAsync( - string topicName, - DateTimeOffset olderThan, - int batchSize, - int initialShardNumber, - string? initialPaginationToken, - CancellationToken cancellationToken) - { - var numShards = _configuration.NumberOfShards <= 1 ? 1 : _configuration.NumberOfShards; - var results = new List(); + await Task.WhenAll(tasks); + + // Set the next page number based on the pagination tokens for the different segments + _dispatchedAllTopicsScanContext.SetNextPage(); + + var allMessages = tasks.SelectMany(t => t.Result); + return allMessages + .OrderBy(m => m.DeliveryTime) + .Select(m => m.ConvertToMessage()); + } + finally + { + _dispatchedAllTopicsScanContext.Release(); + } + } - var paginationToken = initialPaginationToken; - var isDone = false; - var shard = initialShardNumber; - while (shard < numShards && results.Count < batchSize) + private async Task> ScanDispatchedIndexSegment(DateTimeOffset dispatchedBefore, + int pageSize, + int pageNumber, + int segmentNumber, + CancellationToken cancellationToken) { + string? paginationToken = null; + if (pageNumber != 1 && paginationToken == null) + { + // It may be that this segment is done but other segments have more results + return new List(); + } + else if (pageNumber != 1) + { + paginationToken = _dispatchedAllTopicsScanContext.GetPagingToken(segmentNumber); + } + + var segmentMessages = new List(); do { - var queryConfig = new QueryOperationConfig + var scanFilter = new ScanFilter(); + scanFilter.AddCondition("DeliveryTime", ScanOperator.LessThanOrEqual, + new List + { + new AttributeValue() + { + N = dispatchedBefore.Ticks.ToString() + } + }); + var scanConfig = new ScanOperationConfig { - IndexName = _configuration.OutstandingIndexName, - KeyExpression = new KeyTopicOutstandingCreatedTimeExpression().Generate(topicName, olderThan, shard), - Limit = batchSize - results.Count, + IndexName = _configuration.DeliveredAllTopicsIndexName, + ConsistentRead = false, + Filter = scanFilter, + Select = SelectValues.AllProjectedAttributes, + Limit = pageSize - segmentMessages.Count, PaginationToken = paginationToken, - ConsistentRead = false + Segment = segmentNumber, + TotalSegments = _configuration.ScanConcurrency }; + var scan = _context.FromScanAsync(scanConfig, _fromScanConfig); - var asyncSearch = _context.FromQueryAsync(queryConfig, _fromQueryConfig); - results.AddRange(await asyncSearch.GetNextSetAsync(cancellationToken)); + segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); - paginationToken = asyncSearch.PaginationToken; - isDone = asyncSearch.IsDone; - } while (results.Count < batchSize && !isDone); + paginationToken = scan.PaginationToken; + } while (paginationToken != null && segmentMessages.Count < pageSize); - // Only move on to the next shard if we still have room to fill up in the batch - if (results.Count < batchSize) + // If there are more results, store the context for retrieving the next page + if (paginationToken != null) { - shard++; + _dispatchedAllTopicsScanContext.SetPagingToken(segmentNumber, paginationToken); + } + else + { + _dispatchedAllTopicsScanContext.SetPagingToken(segmentNumber, null); } - } - var nextShardNumber = 0; - var queryComplete = true; - if (!isDone) - { - // We are part way through a shard - // continue on the current shard in the next batch - nextShardNumber = shard; - queryComplete = false; + return segmentMessages; } - else if (shard < numShards) + + private async Task PageIndexQueryToBatchSize( + string topicName, + DateTimeOffset sinceTime, + int batchSize, + int initialShardNumber, + string? initialPaginationToken, + string indexName, + TopicQueryKeyExpression keyExpression, + CancellationToken cancellationToken) { - // We are exactly at the end of a shard - // continue on the next shard in the next batch - nextShardNumber = shard + 1; - queryComplete = false; - } + var numShards = _configuration.NumberOfShards <= 1 ? 1 : _configuration.NumberOfShards; - return new OutstandingMessagesQueryResult(results, nextShardNumber, paginationToken, queryComplete); - } + var results = new List(); + var paginationToken = initialPaginationToken; + var isDone = false; + var shard = initialShardNumber; - private async Task PageDispatchedMessagesToBatchSizeAsync( - string topicName, - DateTimeOffset sinceTime, - int batchSize, - string? initialPaginationToken, - CancellationToken cancellationToken) - { - var results = new List(); - var keyExpression = new KeyTopicDeliveredTimeExpression().Generate(topicName, sinceTime); - var paginationToken = initialPaginationToken; - var isDone = false; - do - { - var queryConfig = new QueryOperationConfig - { - IndexName = _configuration.DeliveredIndexName, - KeyExpression = keyExpression, - Limit = batchSize - results.Count, - PaginationToken = paginationToken - }; + while (shard < numShards && results.Count < batchSize) + { + do + { + var queryConfig = new QueryOperationConfig + { + IndexName = indexName, + KeyExpression = keyExpression.Generate(topicName, sinceTime, shard), + Limit = batchSize - results.Count, + PaginationToken = paginationToken + }; + + var asyncSearch = _context.FromQueryAsync(queryConfig, _fromQueryConfig); + results.AddRange(await asyncSearch.GetNextSetAsync(cancellationToken)); + + paginationToken = asyncSearch.PaginationToken; + isDone = asyncSearch.IsDone; + } while (results.Count < batchSize && !isDone); + + // Only move on to the next shard if we still have room to fill up in the batch + if (results.Count < batchSize) + { + shard++; + } + } - var asyncSearch = _context.FromQueryAsync(queryConfig, _fromQueryConfig); - results.AddRange(await asyncSearch.GetNextSetAsync(cancellationToken)); + var nextShardNumber = 0; + var queryComplete = true; + if (!isDone) + { + // We are part way through a shard + // continue on the current shard in the next batch + nextShardNumber = shard; + queryComplete = false; + } + else if (shard < numShards) + { + // We are exactly at the end of a shard + // continue on the next shard in the next batch + nextShardNumber = shard + 1; + queryComplete = false; + } - paginationToken = asyncSearch.PaginationToken; - isDone = asyncSearch.IsDone; - } while (results.Count < batchSize && !isDone); + return new IndexQueryResult(results, nextShardNumber, paginationToken, queryComplete); + } - return new DispatchedMessagesQueryResult(results, paginationToken, isDone); - } - - private async Task WriteMessageToOutbox(CancellationToken cancellationToken, MessageItem messageToStore) - { - await _context.SaveAsync( - messageToStore, - _saveConfig, - cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - } + private async Task WriteMessageToOutbox(CancellationToken cancellationToken, MessageItem messageToStore) + { + await _context.SaveAsync( + messageToStore, + _saveConfig, + cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + } - private int GetShardNumber() - { - if (_configuration.NumberOfShards <= 1) - return 0; + private int GetShardNumber() + { + if (_configuration.NumberOfShards <= 1) + return 0; - //The range is inclusive of 0 but exclusive of NumberOfShards i.e. 0, 4 produces values in range 0-3 - return _random.Next(0, _configuration.NumberOfShards); - } + //The range is inclusive of 0 but exclusive of NumberOfShards i.e. 0, 4 produces values in range 0-3 + return _random.Next(0, _configuration.NumberOfShards); + } - private long? GetExpirationTime() - { - if (_configuration.TimeToLive.HasValue) + private long? GetExpirationTime() { - return _timeProvider.GetUtcNow().Add(_configuration.TimeToLive.Value).ToUnixTimeSeconds(); - } + if (_configuration.TimeToLive.HasValue) + { + return _timeProvider.GetUtcNow().Add(_configuration.TimeToLive.Value).ToUnixTimeSeconds(); + } - return null; + return null; + } } } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingMessagesQueryResult.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/IndexQueryResult.cs similarity index 91% rename from src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingMessagesQueryResult.cs rename to src/Paramore.Brighter.Outbox.DynamoDB.V4/IndexQueryResult.cs index 662f3f8d2a..d1bde8cc9b 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingMessagesQueryResult.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/IndexQueryResult.cs @@ -27,7 +27,7 @@ THE SOFTWARE. */ namespace Paramore.Brighter.Outbox.DynamoDB.V4; -internal sealed class OutstandingMessagesQueryResult(IEnumerable messages, int shardNumber, string? paginationToken, bool queryComplete) +internal sealed class IndexQueryResult(IEnumerable messages, int shardNumber, string? paginationToken, bool queryComplete) { public IEnumerable Messages { get; private set; } = messages; public int ShardNumber { get; private set; } = shardNumber; diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicDeliveredTimeExpression.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicDeliveredTimeExpression.cs index fd9613a839..9bd392a619 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicDeliveredTimeExpression.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicDeliveredTimeExpression.cs @@ -2,28 +2,24 @@ using System.Collections.Generic; using Amazon.DynamoDBv2.DocumentModel; -namespace Paramore.Brighter.Outbox.DynamoDB.V4; - -internal sealed class KeyTopicDeliveredTimeExpression +namespace Paramore.Brighter.Outbox.DynamoDB.V4 { - private readonly Expression _expression = new() - { - ExpressionStatement = "Topic = :v_Topic and DeliveryTime < :v_SinceTime" - }; - - public override string ToString() + internal sealed class KeyTopicDeliveredTimeExpression : TopicQueryKeyExpression { - return _expression.ExpressionStatement; - } + public KeyTopicDeliveredTimeExpression() + { + Expression.ExpressionStatement = "TopicShard = :v_TopicShard and DeliveryTime < :v_SinceTime"; + } - public Expression Generate(string topicName, DateTimeOffset sinceTime) - { - _expression.ExpressionAttributeValues = new Dictionary(capacity: 2) + public override Expression Generate(string topicName, DateTimeOffset sinceTime, int shard) { - { ":v_Topic", topicName }, - { ":v_SinceTime", sinceTime.Ticks } - }; + Expression.ExpressionAttributeValues = new Dictionary(capacity: 2) + { + { ":v_TopicShard", $"{topicName}_{shard}" }, + { ":v_SinceTime", sinceTime.Ticks } + }; - return _expression; + return Expression; + } } -} \ No newline at end of file +} diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicOutstandingCreatedTimeExpression.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicOutstandingCreatedTimeExpression.cs index 86a4798b78..c33e694064 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicOutstandingCreatedTimeExpression.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/KeyTopicOutstandingCreatedTimeExpression.cs @@ -2,28 +2,24 @@ using System.Collections.Generic; using Amazon.DynamoDBv2.DocumentModel; -namespace Paramore.Brighter.Outbox.DynamoDB.V4; - -internal sealed class KeyTopicOutstandingCreatedTimeExpression +namespace Paramore.Brighter.Outbox.DynamoDB.V4 { - private readonly Expression _expression = new() - { - ExpressionStatement = "TopicShard = :v_TopicShard and OutstandingCreatedTime < :v_OutstandingCreatedTime" - }; - - public override string ToString() + internal sealed class KeyTopicOutstandingCreatedTimeExpression : TopicQueryKeyExpression { - return _expression.ExpressionStatement; - } + public KeyTopicOutstandingCreatedTimeExpression() + { + Expression.ExpressionStatement = "TopicShard = :v_TopicShard and OutstandingCreatedTime < :v_OutstandingCreatedTime"; + } - public Expression Generate(string topicName, DateTimeOffset createdTime, int shard) - { - _expression.ExpressionAttributeValues = new Dictionary(capacity: 2) + public override Expression Generate(string topicName, DateTimeOffset createdTime, int shard) { - { ":v_TopicShard", $"{topicName}_{shard}" }, - { ":v_OutstandingCreatedTime", createdTime.Ticks } - }; + Expression.ExpressionAttributeValues = new Dictionary(capacity: 2) + { + { ":v_TopicShard", $"{topicName}_{shard}" }, + { ":v_OutstandingCreatedTime", createdTime.Ticks } + }; - return _expression; + return Expression; + } } -} \ No newline at end of file +} diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingTopicQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingTopicQueryContext.cs deleted file mode 100644 index fcd04e614d..0000000000 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingTopicQueryContext.cs +++ /dev/null @@ -1,33 +0,0 @@ -#region Licence - -/* The MIT License (MIT) -Copyright © 2024 Dominic Hickie - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the “Software”), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. */ - -#endregion - -namespace Paramore.Brighter.Outbox.DynamoDB.V4; - -internal sealed class OutstandingTopicQueryContext(int nextPage, int shardNumber, string? lastEvaluatedKey) -{ - public int NextPage { get; private set; } = nextPage; - public int ShardNumber { get; private set; } = shardNumber; - public string? LastEvaluatedKey { get; private set; } = lastEvaluatedKey; -} diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingAllTopicsQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/TopicQueryContext.cs similarity index 65% rename from src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingAllTopicsQueryContext.cs rename to src/Paramore.Brighter.Outbox.DynamoDB.V4/TopicQueryContext.cs index 89aeccb224..d4c16a169f 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/OutstandingAllTopicsQueryContext.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/TopicQueryContext.cs @@ -23,22 +23,33 @@ THE SOFTWARE. */ #endregion -using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; namespace Paramore.Brighter.Outbox.DynamoDB.V4; -internal sealed class OutstandingAllTopicsQueryContext +internal sealed class TopicQueryContext() { - public int NextPage { get; private set; } - public string LastEvaluatedKey { get; private set; } - public int ShardNumber { get; private set; } - public List RemainingTopics { get; private set; } + public int NextPage { get; private set; } = 1; + public int ShardNumber { get; private set; } = 0; + public string? LastEvaluatedKey { get; private set; } = null; - public OutstandingAllTopicsQueryContext(int nextPage, string lastEvaluatedKey, int shardNumber, List remainingTopics) + private SemaphoreSlim _queryLock = new(1, 1); + + public void SetPaginationState(int nextPage, int shardNumber, string? lastEvaluatedKey) { NextPage = nextPage; - LastEvaluatedKey = lastEvaluatedKey; ShardNumber = shardNumber; - RemainingTopics = remainingTopics; - } + LastEvaluatedKey = lastEvaluatedKey; + } + + public void Reset() + { + NextPage = 1; + ShardNumber = 0; + LastEvaluatedKey = null; + } + + public async Task Lock(CancellationToken cancellationToken) => await _queryLock.WaitAsync(cancellationToken); + public void Release() => _queryLock.Release(); } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedTopicQueryContext.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/TopicQueryKeyExpression.cs similarity index 74% rename from src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedTopicQueryContext.cs rename to src/Paramore.Brighter.Outbox.DynamoDB.V4/TopicQueryKeyExpression.cs index 287f0e4a73..7dc68783f7 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DispatchedTopicQueryContext.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/TopicQueryKeyExpression.cs @@ -1,7 +1,7 @@ #region Licence /* The MIT License (MIT) -Copyright © 2024 Dominic Hickie +Copyright © 2025 Dominic Hickie Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal @@ -23,16 +23,18 @@ THE SOFTWARE. */ #endregion -namespace Paramore.Brighter.Outbox.DynamoDB.V4; +using System; +using Amazon.DynamoDBv2.DocumentModel; -internal sealed class DispatchedTopicQueryContext +namespace Paramore.Brighter.Outbox.DynamoDB.V4; +internal abstract class TopicQueryKeyExpression { - public int NextPage { get; private set; } - public string LastEvaluatedKey { get; private set; } + internal Expression Expression { get; } = new(); - public DispatchedTopicQueryContext(int nextPage, string lastEvaluatedKey) + public override string ToString() { - NextPage = nextPage; - LastEvaluatedKey = lastEvaluatedKey; + return Expression.ExpressionStatement; } + + public abstract Expression Generate(string topicName, DateTimeOffset sinceTime, int shard); } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 0f0a9a4045..f78a3ec7d9 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -350,29 +350,13 @@ public async Task> DispatchedMessagesAsync( return await DispatchedMessagesAsync(dispatchedSince, requestContext, pageSize: pageSize, pageNumber: 1, outboxTimeout: -1, args: null, cancellationToken: cancellationToken); } - /// - /// Finds a message with the specified identifier. - /// Sync over async - /// - /// The identifier. - /// What is the context for this request; used to access the Span - /// Timeout in milliseconds; -1 for default timeout - /// - /// + /// public Message Get(Id messageId, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) { return GetAsync(messageId, requestContext, outBoxTimeout, args).ConfigureAwait(ContinueOnCapturedContext).GetAwaiter().GetResult(); } - /// - /// Finds a message with the specified identifier. - /// - /// The identifier. - /// What is the context for this request; used to access the Span - /// Timeout in milliseconds; -1 for default timeout - /// For outboxes that require additional parameters such as topic, provide an optional arg - /// - /// + /// public async Task GetAsync( Id messageId, RequestContext requestContext, @@ -401,6 +385,46 @@ public async Task GetAsync( } } + /// + public IEnumerable Get(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) + { + return GetAsync(messageIds, requestContext, outBoxTimeout, args).ConfigureAwait(ContinueOnCapturedContext).GetAwaiter().GetResult(); + } + + /// + public async Task> GetAsync( + IEnumerable messageIds, + RequestContext requestContext, + int outBoxTimeout = -1, + Dictionary? args = null, + CancellationToken cancellationToken = default) + { + var dbAttributes = new Dictionary() + { + {"db.operation.parameter.message.ids", string.Join(",", messageIds.Select(x => x.ToString()))} + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Get, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); + + try + { + var batchGet = _context.CreateBatchGet(_dynamoOverwriteTableConfig); + foreach (var id in messageIds) + { + batchGet.AddKey(id.Value); + } + + await batchGet.ExecuteAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + return batchGet.Results.Select(x => x.ConvertToMessage()); + } + finally + { + Tracer?.EndSpan(span); + } + } + /// /// Update a message to show it is dispatched /// diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs b/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs index 45fbdef092..64459d73d0 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/TopicQueryKeyExpression.cs @@ -1,17 +1,40 @@ -using System; +#region Licence + +/* The MIT License (MIT) +Copyright © 2025 Dominic Hickie + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ + +#endregion + +using System; using Amazon.DynamoDBv2.DocumentModel; -namespace Paramore.Brighter.Outbox.DynamoDB +namespace Paramore.Brighter.Outbox.DynamoDB; +internal abstract class TopicQueryKeyExpression { - internal abstract class TopicQueryKeyExpression - { - internal Expression Expression { get; } = new(); - - public override string ToString() - { - return Expression.ExpressionStatement; - } + internal Expression Expression { get; } = new(); - public abstract Expression Generate(string topicName, DateTimeOffset sinceTime, int shard); + public override string ToString() + { + return Expression.ExpressionStatement; } + + public abstract Expression Generate(string topicName, DateTimeOffset sinceTime, int shard); } diff --git a/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs b/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs index 05ba5f5ce9..c13260931e 100644 --- a/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs @@ -102,6 +102,7 @@ public async Task> GetAsync( IEnumerable messageIds, RequestContext? requestContext, int outBoxTimeout = -1, + Dictionary? args = null, CancellationToken cancellationToken = default ) { @@ -558,17 +559,12 @@ public IList Get(int pageSize = 100, int pageNumber = 1, Dictionary - /// Returns messages specified by the Ids - /// - /// The Ids of the messages - /// What is the context for this request; used to access the Span - /// The Timeout of the outbox. - /// + /// public IEnumerable Get( IEnumerable messageIds, RequestContext? requestContext = null, - int outBoxTimeout = -1 + int outBoxTimeout = -1, + Dictionary? args = null ) { var ids = messageIds.Select(id => id.Value).ToArray(); diff --git a/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs b/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs index 8ac328e31d..7b0a93999e 100644 --- a/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs +++ b/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs @@ -252,6 +252,11 @@ public Message Get(Id messageId, RequestContext requestContext, int outBoxTimeou return new Message(){Header = new MessageHeader("",new RoutingKey(""), MessageType.MT_NONE)}; } + public IEnumerable Get(IEnumerable messageId, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) + { + return []; + } + public void MarkDispatched(Id id, RequestContext requestContext, DateTimeOffset? dispatchedAt = null, Dictionary? args = null) { //ignore diff --git a/src/Paramore.Brighter/IAmAnOutboxAsync.cs b/src/Paramore.Brighter/IAmAnOutboxAsync.cs index e07e38aacc..e32d3eb9f7 100644 --- a/src/Paramore.Brighter/IAmAnOutboxAsync.cs +++ b/src/Paramore.Brighter/IAmAnOutboxAsync.cs @@ -106,7 +106,7 @@ Task DeleteAsync( /// Timeout of sql call. /// Additional parameters required for search, if any /// The Cancellation Token - /// List of messages that need to be dispatched. + /// List of messages that have been dispatched. Task> DispatchedMessagesAsync( TimeSpan dispatchedSince, RequestContext requestContext, @@ -134,6 +134,23 @@ Task GetAsync( CancellationToken cancellationToken = default ); + /// + /// Awaitable Get messages with the specified identifiers. + /// + /// The message identifiers + /// The context for the request pipeline; gives us the OTel span for example + /// The time allowed for the read in milliseconds; -1 for default timeout + /// For outboxes that require additional parameters such as topic, provide an optional arg + /// Allows the sender to cancel the request pipeline. Optional + /// . + Task> GetAsync( + IEnumerable messageId, + RequestContext requestContext, + int outBoxTimeout = -1, + Dictionary? args = null, + CancellationToken cancellationToken = default + ); + /// /// Update a message to show it is dispatched /// diff --git a/src/Paramore.Brighter/IAmAnOutboxSync.cs b/src/Paramore.Brighter/IAmAnOutboxSync.cs index be61e88f05..5fc71fef7e 100644 --- a/src/Paramore.Brighter/IAmAnOutboxSync.cs +++ b/src/Paramore.Brighter/IAmAnOutboxSync.cs @@ -72,7 +72,7 @@ public interface IAmAnOutboxSync : IAmAnOutbox where T : Messag /// The page number. /// Timeout of sql call. /// Additional parameters required for search, if any - /// List of messages that need to be dispatched. + /// List of messages that have been dispatched. IEnumerable DispatchedMessages( TimeSpan dispatchedSince, RequestContext requestContext, @@ -86,11 +86,21 @@ IEnumerable DispatchedMessages( /// /// The message identifier. /// What is the context for this request; used to access the Span - /// The time allowed for the read in milliseconds; on a -2 default + /// The time allowed for the read in milliseconds; -1 for default timeout /// For outboxes that require additional parameters such as topic, provide an optional arg - /// Task<Message>. + /// The requested message Message Get(Id messageId, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null); + /// + /// Gets a batch of messages with the specified identifiers. + /// + /// The identifiers of the messages to retrieve. + /// What is the context for this request; used to access the Span + /// The time allowed for the read in milliseconds; -1 for default timeout + /// For outboxes that require additional parameters such as topic, provide an optional arg + /// The requested messages + IEnumerable Get(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null); + /// /// Update a message to show it is dispatched /// diff --git a/src/Paramore.Brighter/InMemoryOutbox.cs b/src/Paramore.Brighter/InMemoryOutbox.cs index 3a93e65dd6..67988e2aa1 100644 --- a/src/Paramore.Brighter/InMemoryOutbox.cs +++ b/src/Paramore.Brighter/InMemoryOutbox.cs @@ -358,7 +358,31 @@ public Message Get(Id messageId, RequestContext requestContext, int outBoxTimeou { Tracer?.EndSpan(span); } + } + + /// + public IEnumerable Get(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, + Dictionary? args = null) + { + ClearExpiredMessages(); + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Brighter, InMemoryAttributes.OutboxDbName, BoxDbOperation.Get, InMemoryAttributes.DbTable), + requestContext?.Span, + options: _instrumentationOptions + ); + + try + { + return messageIds + .Select(id => Requests.TryGetValue(id, out OutboxEntry? entry) ? entry.Message : null) + .Where(msg => msg != null) + .Select(msg => msg!); + } + finally + { + Tracer?.EndSpan(span); + } } /// @@ -393,6 +417,27 @@ public Task GetAsync( return tcs.Task; } + /// + public Task> GetAsync( + IEnumerable messageIds, + RequestContext requestContext, + int outBoxTimeout = -1, + Dictionary? args = null, + CancellationToken cancellationToken = default) + { + //NOTE: We don't create a span here as we just call the sync method + + var tcs = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + if (cancellationToken.IsCancellationRequested) + { + tcs.SetCanceled(); + return tcs.Task; + } + var messages = Get(messageIds, requestContext, outBoxTimeout); + tcs.SetResult(messages); + return tcs.Task; + } + /// /// Mark the message as dispatched /// diff --git a/src/Paramore.Brighter/RelationDatabaseOutbox.cs b/src/Paramore.Brighter/RelationDatabaseOutbox.cs index d8fc6a5943..08deb53010 100644 --- a/src/Paramore.Brighter/RelationDatabaseOutbox.cs +++ b/src/Paramore.Brighter/RelationDatabaseOutbox.cs @@ -573,18 +573,12 @@ public async Task GetAsync( } } - /// - /// Returns messages specified by the Ids - /// - /// The Ids of the messages - /// What is the context for this request; used to access the Span - /// The Timeout of the outbox. - /// Cancellation Token. - /// + /// public async Task> GetAsync( IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, + Dictionary? args = null, CancellationToken cancellationToken = default ) { diff --git a/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs b/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs index 8beaff9c5b..355fd35313 100644 --- a/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs +++ b/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs @@ -75,11 +75,21 @@ public Message Get(Id messageId, RequestContext requestContext, int outBoxTimeou return Messages.First(m => m.Message.Id == messageId).Message; } + public IEnumerable Get(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) + { + return messageIds.Select(id => Get(id, requestContext, outBoxTimeout, args)); + } + public Task GetAsync(Id messageId, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null, CancellationToken cancellationToken = default) { return Task.FromResult(Get(messageId, requestContext, outBoxTimeout, args)); } + public Task> GetAsync(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null, CancellationToken cancellationToken = default) + { + return Task.FromResult(Get(messageIds, requestContext, outBoxTimeout, args)); + } + public void MarkDispatched(Id id, RequestContext requestContext, DateTimeOffset? dispatchedAt = null, Dictionary? args = null) { var entry = Messages.First(m => m.Message.Id == id); From 4077466a225d66c8c6ac8b3320df2417633bfec1 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Wed, 17 Sep 2025 11:32:00 +0100 Subject: [PATCH 08/22] Fix AWS SDK project reference This fixes a project reference so that it uses v4 of the AWS SDK for both references and not just one of them. --- .../AWSTransfomers/CloudEvents/Greetings/Greetings.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/Transforms/AWSTransfomers/CloudEvents/Greetings/Greetings.csproj b/samples/Transforms/AWSTransfomers/CloudEvents/Greetings/Greetings.csproj index 7138682d5a..e2150eb1bc 100644 --- a/samples/Transforms/AWSTransfomers/CloudEvents/Greetings/Greetings.csproj +++ b/samples/Transforms/AWSTransfomers/CloudEvents/Greetings/Greetings.csproj @@ -4,8 +4,8 @@ + - From 7b73ec1a6b677e0e941646fe968ec939f4a48401 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Wed, 17 Sep 2025 12:53:10 +0100 Subject: [PATCH 09/22] Use BatchGet when clearing messages from outbox This updates the OutboxProducerMediator to take advantage of BatchGet operations when clearing a specific collection of IDs from the outbox, to minimise the number of concurrent db operations. --- .../OutboxProducerMediator.cs | 78 +++++++++++-------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/src/Paramore.Brighter/OutboxProducerMediator.cs b/src/Paramore.Brighter/OutboxProducerMediator.cs index 5cd68f6486..a880c2f52a 100644 --- a/src/Paramore.Brighter/OutboxProducerMediator.cs +++ b/src/Paramore.Brighter/OutboxProducerMediator.cs @@ -297,8 +297,7 @@ public void CallViaExternalBus(Message outMessage, RequestContext? public void ClearOutbox( Id[] posts, RequestContext requestContext, - Dictionary? args = null - ) + Dictionary? args = null) { if (!HasOutbox()) throw new InvalidOperationException("No outbox defined."); @@ -309,30 +308,38 @@ public void ClearOutbox( var childSpans = new ConcurrentDictionary(); try { - if (_outBox is null) throw new ArgumentException(NoSyncOutboxError); - foreach (var messageId in posts) + // Get all the messages being cleared in a batch to keep db operations down + var messages = _outBox!.Get(posts, requestContext).ToArray(); + if (messages.Length != posts.Length) + { + var missingMessageIds = posts.Where(id => !messages.Any(m => m.Id == id)); + throw new NullReferenceException($"Message(s) with Id(s) {string.Join(",", missingMessageIds)} not found in Outbox"); + } + BrighterTracer.WriteOutboxEvent(BoxDbOperation.Get, messages, parentSpan, false, false, + _instrumentationOptions); + + foreach (var message in messages) { var span = _tracer?.CreateClearSpan(CommandProcessorSpanOperation.Clear, requestContext.Span, - messageId, _instrumentationOptions); + message.Id, _instrumentationOptions); if (span is not null) { - childSpans.TryAdd(messageId, span); + childSpans.TryAdd(message.Id, span); requestContext.Span = span; } - - var message = _outBox.Get(messageId, requestContext); - if (message is null || message.Header.MessageType == MessageType.MT_NONE) - throw new NullReferenceException($"Message with Id {messageId} not found in the Outbox"); - - BrighterTracer.WriteOutboxEvent(BoxDbOperation.Get, message, span, false, false, - _instrumentationOptions); - Dispatch([message], requestContext, args); + try + { + Dispatch([message], requestContext, args); + } + finally + { + _tracer?.EndSpan(span); + } } } finally { - _tracer?.EndSpans(childSpans); requestContext.Span = parentSpan; } @@ -365,29 +372,38 @@ public async Task ClearOutboxAsync( var childSpans = new ConcurrentDictionary(); try { - if(_asyncOutbox is null)throw new ArgumentException(NoAsyncOutboxError); - foreach (var messageId in posts) + // Get all the messages being cleared in a batch to keep db operations down + Message[] messages = (await _asyncOutbox!.GetAsync(posts, requestContext)).ToArray(); + if (messages.Length != posts.ToArray().Length) { - var span = _tracer?.CreateClearSpan(CommandProcessorSpanOperation.Clear, requestContext.Span, - messageId, _instrumentationOptions); - if (span != null) childSpans.TryAdd(messageId, span); - requestContext.Span = span; - - var message = await _asyncOutbox.GetAsync(messageId, requestContext, _outboxTimeout, args, - cancellationToken); - if (message is null || message.Header.MessageType == MessageType.MT_NONE) - throw new NullReferenceException($"Message with Id {messageId} not found in the Outbox"); - - BrighterTracer.WriteOutboxEvent(BoxDbOperation.Get, message, requestContext.Span, false, true, + var missingMessageIds = posts.Where(id => !messages.Any(m => m.Id == id)); + throw new NullReferenceException($"Message(s) with Id(s) {string.Join(",", missingMessageIds)} not found in Outbox"); + } + BrighterTracer.WriteOutboxEvent(BoxDbOperation.Get, messages, parentSpan, false, true, _instrumentationOptions); - await DispatchAsync([message], requestContext, continueOnCapturedContext, - cancellationToken); + foreach (var message in messages) + { + var span = _tracer?.CreateClearSpan(CommandProcessorSpanOperation.Clear, requestContext.Span, + message.Id, _instrumentationOptions); + if (span != null) + { + childSpans.TryAdd(message.Id, span); + requestContext.Span = span; + } + + try + { + await DispatchAsync([message], requestContext, continueOnCapturedContext, cancellationToken); + } + finally + { + _tracer?.EndSpan(span); + } } } finally { - _tracer?.EndSpans(childSpans); requestContext.Span = parentSpan; } From 4eb168aabff74c22427951bca6ad0d1b6570889a Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Fri, 19 Sep 2025 14:07:58 +0100 Subject: [PATCH 10/22] Use update expression to mark messages as dispatched This changes `MarkDispatchedAsync` to do so using an Update expression. This enables the ability to do it using a single call to the DB, instead of reading everything out and writing it all back in again. To facilitate this, a reference to the Dynamo DB client has been added as a member variable, as update expressions can only be used when using the low level API. This has also meant deleting one of the (unused) constructors that takes the Dynamo DB context directly, as the client is not accessible from within the context object. Finally, the ADR has been updated to reflect this change as this wasn't the original proposed change. --- ...ise-reads-and-writes-from-dynamo-outbox.md | 9 +-- .../DynamoDbOutbox.cs | 76 ++++++------------- .../DynamoDbOutbox.cs | 65 ++++++---------- 3 files changed, 50 insertions(+), 100 deletions(-) diff --git a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md index 667ec7f7ee..3359ecf189 100644 --- a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md +++ b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md @@ -27,7 +27,6 @@ As the `Delivered` index isn't sharded, it can fall victim to hot partitions. Wh There are a few operations where the outbox is provided with a collection of messages or message IDs, and instead of performing batch operations with those IDs it iterates through them and performs individual operations sequentially: * When clearing a collection of message IDs from the outbox, it fetches each of these individually and tries to dispatch it before moving onto the next -* When marking a collection of messages as dispatched, the messages are written to sequentially with seperate requests * When a collection of messages are deleted from the outbox they're worked through sequentially with separate requests ### Fetching outstanding message count @@ -36,7 +35,7 @@ Every time one or more messages are cleared from the outbox, the `OutboxProducer ## Decision -All of the inefficiencies above can be improved with non-breaking changes. +Some of the inefficiencies above can be improved with non-breaking changes. There will, however, be breaking changes required to the delivered index. ### `OutstandingMessages` operation @@ -72,11 +71,7 @@ Update `OutboxProducerMediator` to use the new `Get` methods. ### Marking messages as dispatched -When marking a collection of messages as dispatched, use a `BatchWrite` operation to update all of them at once. - -By default, if any of the updates fail throw an exception. If, however, an option is passed in the `args` dictionary with the key `LogBatchErrors` and a value of `true`, then don't throw and instead log a warning message. When the method is invoked from the outbox sweeper, pass this new option to the method. - -Add a new overload of `MarkDispatched` which takes a collection of message IDs (currently the only bulk option is the async version). +When marking a message as dispatched, use an `UpdateExpression` to only update the attributes we need to instead of reading the whole message out and then writing it all back in again. ### Deleting messages from the outbox diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index e0b5e05b96..e90b6c6065 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -44,6 +44,7 @@ public class DynamoDbOutbox : { private readonly DynamoDbConfiguration _configuration; private readonly DynamoDBContext _context; + private readonly IAmazonDynamoDB _client; private readonly LoadConfig _loadConfig; private readonly BatchGetConfig _batchGetConfig; private readonly SaveConfig _saveConfig; @@ -86,6 +87,7 @@ public DynamoDbOutbox( InstrumentationOptions instrumentationOptions = InstrumentationOptions.All) { _configuration = configuration; + _client = client; _context = new DynamoDBContextBuilder().WithDynamoDBClient(() => client).Build(); _timeProvider = timeProvider ?? TimeProvider.System; _loadConfig = new LoadConfig @@ -115,41 +117,6 @@ public DynamoDbOutbox( _instrumentationOptions = instrumentationOptions; } - /// - /// Initialises a new instance of the class. - /// - /// An existing Dynamo Db Context - /// The Configuration from the context - the config is internal, so we can't grab the settings from it. - public DynamoDbOutbox(DynamoDBContext context, DynamoDbConfiguration configuration, TimeProvider timeProvider) - { - _context = context; - _configuration = configuration; - _timeProvider = timeProvider; - _loadConfig = new LoadConfig - { - OverrideTableName = _configuration.TableName, - ConsistentRead = true - }; - - _saveConfig = new SaveConfig { OverrideTableName = _configuration.TableName }; - _deleteConfig = new DeleteConfig { OverrideTableName = _configuration.TableName, }; - _fromQueryConfig = new FromQueryConfig { OverrideTableName = _configuration.TableName }; - _fromScanConfig = new FromScanConfig { OverrideTableName = _configuration.TableName }; - _toDocumentConfig = new ToDocumentConfig { OverrideTableName = _configuration.TableName, }; - _batchGetConfig = new BatchGetConfig { OverrideTableName = _configuration.TableName }; - - if (_configuration.NumberOfShards > 20) - { - throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); - } - - _outstandingTopicQueryContexts = new ConcurrentDictionary(); - _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - - _outstandingAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); - _dispatchedAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); - } - /// /// /// Adds a message to the Outbox @@ -475,14 +442,29 @@ public async Task MarkDispatchedAsync( try { - var message = await _context.LoadAsync(id.Value, _loadConfig, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - MarkMessageDispatched(dispatchedAt ?? _timeProvider.GetUtcNow(), message); + var dispatchTime = dispatchedAt ?? _timeProvider.GetUtcNow(); + var updateItemRequest = new UpdateItemRequest() + { + TableName = _configuration.TableName, + Key = new Dictionary + { + { "MessageId", new AttributeValue { S = id } } + }, + // Remove the outstanding created time attribute to remove it from the outstanding index + UpdateExpression = "SET DeliveryTime = :deliveryTime, DeliveredAt = :deliveredAt REMOVE OutstandingCreatedTime", + ConditionExpression = "attribute_exists(MessageId)", + ExpressionAttributeValues = new Dictionary() + { + {":deliveryTime", new AttributeValue { N = dispatchTime.Ticks.ToString() } }, + {":deliveredAt", new AttributeValue { S = dispatchTime.ToString("yyyy-MM-ddTHH:mm:ss.fffZ") } } + } + }; - await _context.SaveAsync( - message, - _saveConfig, - cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + await _client.UpdateItemAsync(updateItemRequest, cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + } + catch (ConditionalCheckFailedException e) + { + throw new NullReferenceException($"The message with id {id.Value} could not be found in the outbox", e); } finally { @@ -526,16 +508,6 @@ public void MarkDispatched(Id id, RequestContext requestContext, DateTimeOffset? .GetResult(); } - private static void MarkMessageDispatched(DateTimeOffset dispatchedAt, MessageItem message) - { - message.DeliveryTime = dispatchedAt.Ticks; - message.DeliveredAt = dispatchedAt.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); - - // Set the outstanding created time to null to remove the attribute - // from the item in dynamo - message.OutstandingCreatedTime = null; - } - /// /// Returns messages that have yet to be dispatched /// Sync over async diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index f78a3ec7d9..d527154f2f 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -43,6 +43,7 @@ public class DynamoDbOutbox : IAmAnOutboxAsync { private readonly DynamoDbConfiguration _configuration; + private readonly IAmazonDynamoDB _client; private readonly DynamoDBContext _context; private readonly DynamoDBOperationConfig _dynamoOverwriteTableConfig; private readonly Random _random = new Random(); @@ -80,6 +81,7 @@ public DynamoDbOutbox( InstrumentationOptions instrumentationOptions = InstrumentationOptions.All) { _configuration = configuration; + _client = client; _context = new DynamoDBContext(client); _timeProvider = timeProvider ?? TimeProvider.System; _dynamoOverwriteTableConfig = new DynamoDBOperationConfig @@ -102,30 +104,6 @@ public DynamoDbOutbox( _instrumentationOptions = instrumentationOptions; } - /// - /// Initialises a new instance of the class. - /// - /// An existing Dynamo Db Context - /// The Configuration from the context - the config is internal, so we can't grab the settings from it. - public DynamoDbOutbox(DynamoDBContext context, DynamoDbConfiguration configuration, TimeProvider timeProvider) - { - _context = context; - _configuration = configuration; - _timeProvider = timeProvider; - _dynamoOverwriteTableConfig = new DynamoDBOperationConfig { OverrideTableName = _configuration.TableName }; - - if (_configuration.NumberOfShards > 20) - { - throw new ArgumentOutOfRangeException(nameof(DynamoDbConfiguration.NumberOfShards), "Maximum number of shards is 20"); - } - - _outstandingTopicQueryContexts = new ConcurrentDictionary(); - _dispatchedTopicQueryContexts = new ConcurrentDictionary(); - - _outstandingAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); - _dispatchedAllTopicsScanContext = new AllTopicsScanContext(_configuration.ScanConcurrency); - } - /// /// /// Adds a message to the Outbox @@ -451,14 +429,29 @@ public async Task MarkDispatchedAsync( try { - var message = await _context.LoadAsync(id.Value, _dynamoOverwriteTableConfig, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - MarkMessageDispatched(dispatchedAt ?? _timeProvider.GetUtcNow(), message); + var dispatchTime = dispatchedAt ?? _timeProvider.GetUtcNow(); + var updateItemRequest = new UpdateItemRequest() + { + TableName = _configuration.TableName, + Key = new Dictionary + { + { "MessageId", new AttributeValue { S = id } } + }, + // Remove the outstanding created time attribute to remove it from the outstanding index + UpdateExpression = "SET DeliveryTime = :deliveryTime, DeliveredAt = :deliveredAt REMOVE OutstandingCreatedTime", + ConditionExpression = "attribute_exists(MessageId)", + ExpressionAttributeValues = new Dictionary() + { + {":deliveryTime", new AttributeValue { N = dispatchTime.Ticks.ToString() } }, + {":deliveredAt", new AttributeValue { S = dispatchTime.ToString("yyyy-MM-ddTHH:mm:ss.fffZ") } } + } + }; - await _context.SaveAsync( - message, - _dynamoOverwriteTableConfig, - cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + await _client.UpdateItemAsync(updateItemRequest, cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + } + catch (ConditionalCheckFailedException e) + { + throw new NullReferenceException($"The message with id {id.Value} could not be found in the outbox", e); } finally { @@ -502,16 +495,6 @@ public void MarkDispatched(Id id, RequestContext requestContext, DateTimeOffset? .GetResult(); } - private static void MarkMessageDispatched(DateTimeOffset dispatchedAt, MessageItem message) - { - message.DeliveryTime = dispatchedAt.Ticks; - message.DeliveredAt = dispatchedAt.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); - - // Set the outstanding created time to null to remove the attribute - // from the item in dynamo - message.OutstandingCreatedTime = null; - } - /// /// Returns messages that have yet to be dispatched /// Sync over async From 7e8d087883e17621f5c530d045c1a050e05477bf Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Fri, 19 Sep 2025 15:03:25 +0100 Subject: [PATCH 11/22] Use BatchWrites for deleting from outbox This updates `DeleteAsync` to use batch write operations to delete items in batches of up to 25 items. It does this using the low level API as the method exposed by `DynamoDBContext` doesn't expose a response object with the unprocessed items. --- .../DynamoDbOutbox.cs | 75 ++++++++++++------- .../DynamoDbOutbox.cs | 75 ++++++++++++------- 2 files changed, 96 insertions(+), 54 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index e90b6c6065..69be44e0a6 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -247,9 +247,55 @@ public async Task DeleteAsync( Dictionary? args = null, CancellationToken cancellationToken = default) { - foreach (var messageId in messageIds) + var dbAttributes = new Dictionary() + { + { "db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.ToString())) } + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Delete, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); + + try { - await DeleteAsync(messageId, requestContext, args, cancellationToken); + // Batch writes can only do 25 items at a time + var batches = messageIds + .Select((id, index) => new { id, index }) + .GroupBy(x => x.index / 25) + .Select(g => g.Select(x => x.id).ToList()) + .ToList(); + + foreach (var batch in batches) + { + var writeRequests = batch.Select(id => + new WriteRequest + { + DeleteRequest = new DeleteRequest + { + Key = new Dictionary + { + { "MessageId", new AttributeValue { S = id.Value } } + } + } + }).ToList(); + var request = new BatchWriteItemRequest + { + RequestItems = new Dictionary>() + { + { _configuration.TableName, writeRequests } + } + }; + + var response = await _client.BatchWriteItemAsync(request, cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + if (response.UnprocessedItems.Any()) + { + throw new NullReferenceException($"The messages with ids {string.Join(",", messageIds.Select(id => id.ToString()))} could not be found in the outbox"); + } + } + } + finally + { + Tracer?.EndSpan(span); } } @@ -578,31 +624,6 @@ public async Task> OutstandingMessagesAsync( } } - private async Task DeleteAsync( - Id messageId, - RequestContext? requestContext, - Dictionary? args, - CancellationToken cancellationToken) - { - var dbAttributes = new Dictionary() - { - {"db.operation.parameter.message.id", messageId} - }; - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Delete, _configuration.TableName, dbAttributes: dbAttributes), - requestContext?.Span, - options: _instrumentationOptions); - - try - { - await _context.DeleteAsync(messageId.Value, _deleteConfig, cancellationToken); - } - finally - { - Tracer?.EndSpan(span); - } - } - private async Task> OutstandingMessagesForAllTopicsAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, CancellationToken cancellationToken) { // Only allow one outstanding messages scan at a time to ensure consistency of pagination tokens diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index d527154f2f..32640e94f6 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -234,9 +234,55 @@ public async Task DeleteAsync( Dictionary? args = null, CancellationToken cancellationToken = default) { - foreach (var messageId in messageIds) + var dbAttributes = new Dictionary() + { + { "db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.ToString())) } + }; + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Delete, _configuration.TableName, dbAttributes: dbAttributes), + requestContext?.Span, + options: _instrumentationOptions); + + try { - await DeleteAsync(messageId, requestContext, args, cancellationToken); + // Batch writes can only do 25 items at a time + var batches = messageIds + .Select((id, index) => new { id, index }) + .GroupBy(x => x.index / 25) + .Select(g => g.Select(x => x.id).ToList()) + .ToList(); + + foreach (var batch in batches) + { + var writeRequests = batch.Select(id => + new WriteRequest + { + DeleteRequest = new DeleteRequest + { + Key = new Dictionary + { + { "MessageId", new AttributeValue { S = id.Value } } + } + } + }).ToList(); + var request = new BatchWriteItemRequest + { + RequestItems = new Dictionary>() + { + { _configuration.TableName, writeRequests } + } + }; + + var response = await _client.BatchWriteItemAsync(request, cancellationToken).ConfigureAwait(ContinueOnCapturedContext); + if (response.UnprocessedItems.Any()) + { + throw new NullReferenceException($"The messages with ids {string.Join(",", messageIds.Select(id => id.ToString()))} could not be found in the outbox"); + } + } + } + finally + { + Tracer?.EndSpan(span); } } @@ -565,31 +611,6 @@ public async Task> OutstandingMessagesAsync( } } - private async Task DeleteAsync( - Id messageId, - RequestContext? requestContext, - Dictionary? args, - CancellationToken cancellationToken) - { - var dbAttributes = new Dictionary() - { - {"db.operation.parameter.message.id", messageId} - }; - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.Delete, _configuration.TableName, dbAttributes: dbAttributes), - requestContext?.Span, - options: _instrumentationOptions); - - try - { - await _context.DeleteAsync(messageId.Value, _dynamoOverwriteTableConfig, cancellationToken); - } - finally - { - Tracer?.EndSpan(span); - } - } - private async Task> OutstandingMessagesForAllTopicsAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, CancellationToken cancellationToken) { // Only allow one outstanding messages scan at a time to ensure consistency of pagination tokens From 80ed02328dacca3d1d6f1ed97c2793a9b30e5646 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Tue, 23 Sep 2025 13:06:04 +0100 Subject: [PATCH 12/22] Add outstanding count method to outboxes This commit adds a function to fetch the count of outstanding messages to both sync and async outbox interfaces. Most implementations just proxy the pre-existing Outstanding functions and return the count of the collection, but the Dynamo DB implementation uses the low level API to perform a count using a Scan. It also makes a change to the OutboxProducerMediator to pass a page size when calling the new method. Otherwise, if the user has specified a max outstanding message count that is above the default page size, and they are using an outbox implementation that honours the provided page size, then the max size check can never be tripped. If no max outstanding message count has been specified, then it uses the default value. --- ...ise-reads-and-writes-from-dynamo-outbox.md | 10 +-- .../DynamoDbOutbox.cs | 88 ++++++++++++++++++- .../DynamoDbOutbox.cs | 88 ++++++++++++++++++- .../MongoDbOutbox.cs | 12 +++ .../ControlBus/ControlBusReceiverBuilder.cs | 5 ++ src/Paramore.Brighter/IAmAnOutboxAsync.cs | 16 ++++ src/Paramore.Brighter/IAmAnOutboxSync.cs | 14 +++ src/Paramore.Brighter/InMemoryOutbox.cs | 44 +++++++++- .../Observability/BoxDbOperation.cs | 15 ++-- .../OutboxProducerMediator.cs | 30 +++++-- .../ProducersConfiguration.cs | 4 +- .../RelationDatabaseOutbox.cs | 15 +++- .../TestDoubles/SpyOutbox.cs | 10 +++ 13 files changed, 321 insertions(+), 30 deletions(-) diff --git a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md index 3359ecf189..703e9f8a7e 100644 --- a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md +++ b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md @@ -79,14 +79,12 @@ When deleting a collection of messages from the outbox, do so using a `BatchWrit ### Outstanding item count -It feels useful to have the number of outstanding messages available in a metric. Dynamo DB doesn't have a `Count` operation, but we can get this information without having to pull all messages into memory: +It feels useful to have the number of outstanding messages available in a metric. Dynamo DB doesn't have a `Count` operation, but it does allow `Scan` operations that return only the count of items scanned, minimising the amount of data sent over the wire. As this is still a scan, we still need to specify a page size when this method is invoked: -1. When messages are added to the Dynamo DB outbox, add a new binary attribute containing a single bit of data. -2. Add new methods to the outbox interfaces for `GetOutstandingMessageCount` and `GetOutstandingMessageCountAsync`. -3. In the Dynamo DB implementation, perform a `Scan` operation on the `Outstanding` index, with configuration to only retrieve the new binary attribute. -4. Page through results as required, and sum the total number of records returned. +* If the outbox has a maximum outstanding message count configured, then the page size should be 1 larger than the maximum to ensure the count retreived is at least as big as the configured maximum +* If the outbox does _not_ have a maximum outstanding message count configured, use the default value -This minimises the amount of data sent over the wire, minimises memory consumption, and maximises the number of records returned in each page. +Add a new method to the outbox interfaces called called `GetOutstandingMessageCount` and `GetOutstandingMessageCountAsync` that is called from the `OutboxProducerMediator`. Other outbox implementations can continue to use their implementations of `OutstandingMessages` for now. diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index 69be44e0a6..4033e29bdf 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -624,6 +624,90 @@ public async Task> OutstandingMessagesAsync( } } + /// + public int GetOutstandingMessageCount( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int maxCount = 100, + Dictionary? args = null) + { + return GetOutstandingMessageCountAsync(dispatchedSince, requestContext, maxCount, args) + .ConfigureAwait(ContinueOnCapturedContext) + .GetAwaiter() + .GetResult(); + } + + /// + public async Task GetOutstandingMessageCountAsync( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int maxCount = 100, + Dictionary? args = null, + CancellationToken cancellationToken = default) + { + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.OutStandingMessageCount, _configuration.TableName), + requestContext?.Span, + options: _instrumentationOptions); + + try + { + var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; + + // Spin off requests to scan each segment + var tasks = new List>(); + var segmentMaxCount = maxCount / _configuration.ScanConcurrency; + for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) + { + tasks.Add(ScanOutstandingIndexSegmentForCount(olderThan, segmentMaxCount, segmentNumber, cancellationToken)); + } + + await Task.WhenAll(tasks); + + var totalCount = tasks.Sum(t => t.Result); + span?.AddTag("db.response.returned_rows", totalCount); + return totalCount; + } + finally + { + Tracer?.EndSpan(span); + } + } + + private async Task ScanOutstandingIndexSegmentForCount(DateTimeOffset olderThan, + int maxCount, + int segmentNumber, + CancellationToken cancellationToken) + { + var segmentCount = 0; + Dictionary? lastEvaluatedKey = null; + do + { + var request = new ScanRequest + { + TableName = _configuration.TableName, + IndexName = _configuration.OutstandingAllTopicsIndexName, + ConsistentRead = true, + Limit = maxCount - segmentCount, + ExclusiveStartKey = lastEvaluatedKey, + Segment = segmentNumber, + TotalSegments = _configuration.ScanConcurrency, + Select = Select.COUNT, + ExpressionAttributeValues = new Dictionary + { + {":olderThan", new AttributeValue { N = olderThan.Ticks.ToString() } } + }, + FilterExpression = "OutstandingCreatedTime <= :olderThan" + }; + ScanResponse response = await _client.ScanAsync(request, cancellationToken); + + lastEvaluatedKey = response.LastEvaluatedKey; + segmentCount += response.Count ?? 0; + } while (lastEvaluatedKey != null && lastEvaluatedKey.Keys.Any() && segmentCount < maxCount); + + return segmentCount; + } + private async Task> OutstandingMessagesForAllTopicsAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, CancellationToken cancellationToken) { // Only allow one outstanding messages scan at a time to ensure consistency of pagination tokens @@ -645,7 +729,7 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti var segmentPageSize = pageSize / _configuration.ScanConcurrency; for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanOutstandingIndexSegment(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + tasks.Add(ScanOutstandingIndexSegmentForMessages(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); @@ -664,7 +748,7 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti } } - private async Task> ScanOutstandingIndexSegment(DateTimeOffset olderThan, + private async Task> ScanOutstandingIndexSegmentForMessages(DateTimeOffset olderThan, int pageSize, int pageNumber, int segmentNumber, diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 32640e94f6..026f086b8c 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -611,6 +611,90 @@ public async Task> OutstandingMessagesAsync( } } + /// + public int GetOutstandingMessageCount( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int maxCount = 100, + Dictionary? args = null) + { + return GetOutstandingMessageCountAsync(dispatchedSince, requestContext, maxCount, args) + .ConfigureAwait(ContinueOnCapturedContext) + .GetAwaiter() + .GetResult(); + } + + /// + public async Task GetOutstandingMessageCountAsync( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int maxCount = 100, + Dictionary? args = null, + CancellationToken cancellationToken = default) + { + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Dynamodb, DYNAMO_DB_NAME, BoxDbOperation.OutStandingMessageCount, _configuration.TableName), + requestContext?.Span, + options: _instrumentationOptions); + + try + { + var olderThan = _timeProvider.GetUtcNow() - dispatchedSince; + + // Spin off requests to scan each segment + var tasks = new List>(); + var segmentMaxCount = maxCount / _configuration.ScanConcurrency; + for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) + { + tasks.Add(ScanOutstandingIndexSegmentForCount(olderThan, segmentMaxCount, segmentNumber, cancellationToken)); + } + + await Task.WhenAll(tasks); + + var totalCount = tasks.Sum(t => t.Result); + span?.AddTag("db.response.returned_rows", totalCount); + return totalCount; + } + finally + { + Tracer?.EndSpan(span); + } + } + + private async Task ScanOutstandingIndexSegmentForCount(DateTimeOffset olderThan, + int maxCount, + int segmentNumber, + CancellationToken cancellationToken) + { + var segmentCount = 0; + Dictionary? lastEvaluatedKey = null; + do + { + var request = new ScanRequest + { + TableName = _configuration.TableName, + IndexName = _configuration.OutstandingAllTopicsIndexName, + ConsistentRead = true, + Limit = maxCount - segmentCount, + ExclusiveStartKey = lastEvaluatedKey, + Segment = segmentNumber, + TotalSegments = _configuration.ScanConcurrency, + Select = Select.COUNT, + ExpressionAttributeValues = new Dictionary + { + {":olderThan", new AttributeValue { N = olderThan.Ticks.ToString() } } + }, + FilterExpression = "OutstandingCreatedTime <= :olderThan" + }; + var response = await _client.ScanAsync(request, cancellationToken); + + lastEvaluatedKey = response.LastEvaluatedKey; + segmentCount += response.Count; + } while (lastEvaluatedKey != null && lastEvaluatedKey.Keys.Any() && segmentCount < maxCount); + + return segmentCount; + } + private async Task> OutstandingMessagesForAllTopicsAsync(TimeSpan dispatchedSince, int pageSize, int pageNumber, CancellationToken cancellationToken) { // Only allow one outstanding messages scan at a time to ensure consistency of pagination tokens @@ -632,7 +716,7 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti var segmentPageSize = pageSize / _configuration.ScanConcurrency; for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanOutstandingIndexSegment(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + tasks.Add(ScanOutstandingIndexSegmentForMessages(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); @@ -651,7 +735,7 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti } } - private async Task> ScanOutstandingIndexSegment(DateTimeOffset olderThan, + private async Task> ScanOutstandingIndexSegmentForMessages(DateTimeOffset olderThan, int pageSize, int pageNumber, int segmentNumber, diff --git a/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs b/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs index c13260931e..595882760e 100644 --- a/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.MongoDb/MongoDbOutbox.cs @@ -888,4 +888,16 @@ public IEnumerable OutstandingMessages(TimeSpan dispatchedSince, Reques Tracer?.EndSpan(span); } } + + /// + public int GetOutstandingMessageCount(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null) + { + return OutstandingMessages(dispatchedSince, requestContext, maxCount, 1, null, args).Count(); + } + + /// + public async Task GetOutstandingMessageCountAsync(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null, CancellationToken cancellationToken = default) + { + return (await OutstandingMessagesAsync(dispatchedSince, requestContext, maxCount, 1, null, args, cancellationToken)).Count(); + } } diff --git a/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs b/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs index 7b0a93999e..99fd65ed66 100644 --- a/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs +++ b/src/Paramore.Brighter.ServiceActivator/ControlBus/ControlBusReceiverBuilder.cs @@ -290,6 +290,11 @@ public IEnumerable OutstandingMessages(TimeSpan dispatchedSince) { return []; } + + public int GetOutstandingMessageCount(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null) + { + return 0; + } } } diff --git a/src/Paramore.Brighter/IAmAnOutboxAsync.cs b/src/Paramore.Brighter/IAmAnOutboxAsync.cs index e32d3eb9f7..a526cb193e 100644 --- a/src/Paramore.Brighter/IAmAnOutboxAsync.cs +++ b/src/Paramore.Brighter/IAmAnOutboxAsync.cs @@ -200,5 +200,21 @@ Task> OutstandingMessagesAsync( IEnumerable? trippedTopics = null, Dictionary? args = null, CancellationToken cancellationToken = default); + + /// + /// Gets the current count of outstanding messages in the Outbox + /// + /// The age a message must be before it's considered to be outstanding + /// What is the context for this request; used to access the Span + /// For outbox implementations that have to scan through messages, the maximum number of messages to scan before returning a value + /// Additional parameters required for search, if any + /// Async Cancellation Token + /// The number of outstanding messages in the outbox + Task GetOutstandingMessageCountAsync( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int maxCount = 100, + Dictionary? args = null, + CancellationToken cancellationToken = default); } } diff --git a/src/Paramore.Brighter/IAmAnOutboxSync.cs b/src/Paramore.Brighter/IAmAnOutboxSync.cs index 5fc71fef7e..64d3d41df7 100644 --- a/src/Paramore.Brighter/IAmAnOutboxSync.cs +++ b/src/Paramore.Brighter/IAmAnOutboxSync.cs @@ -127,5 +127,19 @@ IEnumerable OutstandingMessages( int pageNumber = 1, IEnumerable? trippedTopics = null, Dictionary? args = null); + + /// + /// Gets the current count of outstanding messages in the Outbox + /// + /// The age a message must be before it's considered to be outstanding + /// What is the context for this request; used to access the Span + /// For outbox implementations that have to scan through messages, the maximum number of messages to scan before returning a value + /// Additional parameters required for search, if any + /// The number of outstanding messages in the outbox + int GetOutstandingMessageCount( + TimeSpan dispatchedSince, + RequestContext? requestContext, + int maxCount = 100, + Dictionary? args = null); } } diff --git a/src/Paramore.Brighter/InMemoryOutbox.cs b/src/Paramore.Brighter/InMemoryOutbox.cs index 67988e2aa1..8863407f70 100644 --- a/src/Paramore.Brighter/InMemoryOutbox.cs +++ b/src/Paramore.Brighter/InMemoryOutbox.cs @@ -599,7 +599,49 @@ public Task> OutstandingMessagesAsync( return tcs.Task; } - + + /// + public int GetOutstandingMessageCount(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null) + { + ClearExpiredMessages(); + + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Brighter, InMemoryAttributes.OutboxDbName, BoxDbOperation.OutStandingMessageCount, + InMemoryAttributes.DbTable), + requestContext?.Span, + options: _instrumentationOptions); + + try + { + var now = _timeProvider.GetUtcNow(); + var sentBefore = now - dispatchedSince; + var outstandingMessageCount = Requests.Values + .OrderBy(oe => oe.Message.Header.TimeStamp) + .Where(oe => + oe.TimeFlushed == DateTimeOffset.MinValue + && oe.WriteTime <= sentBefore.DateTime) + .Take(maxCount) + .Count(); + return outstandingMessageCount; + } + finally + { + Tracer?.EndSpan(span); + } + } + + /// + public Task GetOutstandingMessageCountAsync(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null, CancellationToken cancellationToken = default) + { + //NOTE: We don't create a span here as we just call the sync method + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + tcs.SetResult(GetOutstandingMessageCount(dispatchedSince, requestContext, maxCount, args)); + + return tcs.Task; + } + private void Delete(Id messageId, RequestContext? requestContext = null) { var span = Tracer?.CreateDbSpan( diff --git a/src/Paramore.Brighter/Observability/BoxDbOperation.cs b/src/Paramore.Brighter/Observability/BoxDbOperation.cs index d37f3f7352..20cc39218c 100644 --- a/src/Paramore.Brighter/Observability/BoxDbOperation.cs +++ b/src/Paramore.Brighter/Observability/BoxDbOperation.cs @@ -29,11 +29,12 @@ namespace Paramore.Brighter.Observability; /// public enum BoxDbOperation { - Add = 0, //Add a message to the inbox/outbox - Delete, //Delete a message from the outbox - DispatchedMessages, //Retrieve a set of messages marked as dispatched - Get, //Get a message from the inbox/outbox by id - MarkDispatched, //Mark one or more messages as dispatched - OutStandingMessages, //Retrieve a set of messages that are still outstanding - Exists //Check whether a message exists in the inbox + Add = 0, //Add a message to the inbox/outbox + Delete, //Delete a message from the outbox + DispatchedMessages, //Retrieve a set of messages marked as dispatched + Get, //Get a message from the inbox/outbox by id + MarkDispatched, //Mark one or more messages as dispatched + OutStandingMessages, //Retrieve a set of messages that are still outstanding + OutStandingMessageCount, //Count how many outstanding messages are in the outbox + Exists //Check whether a message exists in the inbox } diff --git a/src/Paramore.Brighter/OutboxProducerMediator.cs b/src/Paramore.Brighter/OutboxProducerMediator.cs index a880c2f52a..158b335d7b 100644 --- a/src/Paramore.Brighter/OutboxProducerMediator.cs +++ b/src/Paramore.Brighter/OutboxProducerMediator.cs @@ -1016,13 +1016,26 @@ private void OutstandingMessagesCheck(RequestContext? requestContext) { if (_outBox != null) { - _outStandingCount = _outBox - .OutstandingMessages( - _maxOutStandingCheckInterval, - requestContext, - args: _outBoxBag - ) - .Count(); + if (_maxOutStandingMessages >= 0) + { + _outStandingCount = _outBox + .GetOutstandingMessageCount( + _maxOutStandingCheckInterval, + requestContext, + _maxOutStandingMessages + 1, + args: _outBoxBag + ); + } + else + { + _outStandingCount = _outBox + .GetOutstandingMessageCount( + _maxOutStandingCheckInterval, + requestContext, + args: _outBoxBag + ); + } + return; } @@ -1030,8 +1043,7 @@ private void OutstandingMessagesCheck(RequestContext? requestContext) } catch (Exception ex) { - //if we can't talk to the outbox, we would swallow the exception on this thread - //by setting the _outstandingCount to -1, we force an exception + //if we can't talk to the outbox, swallow the exception on this thread Log.ErrorGettingOutstandingMessageCount(s_logger, ex); _outStandingCount = 0; } diff --git a/src/Paramore.Brighter/ProducersConfiguration.cs b/src/Paramore.Brighter/ProducersConfiguration.cs index 4b02071607..ca3d6702b8 100644 --- a/src/Paramore.Brighter/ProducersConfiguration.cs +++ b/src/Paramore.Brighter/ProducersConfiguration.cs @@ -81,7 +81,7 @@ public interface IAmProducersConfiguration /// 0 => No outstanding messages, i.e. throw an error as soon as something goes into the Outbox /// 1+ => Allow this number of messages to stack up in an Outbox before throwing an exception (likely to fail fast) /// - public int? MaxOutStandingMessages { get; set; } + public int MaxOutStandingMessages { get; set; } /// /// At what interval should we check the number of outstanding messages has not exceeded the limit set in MaxOutStandingMessages @@ -192,7 +192,7 @@ public class ProducersConfiguration : IAmProducersConfiguration /// 0 => No outstanding messages, i.e. throw an error as soon as something goes into the Outbox /// 1+ => Allow this number of messages to stack up in an Outbox before throwing an exception (likely to fail fast) /// - public int? MaxOutStandingMessages { get; set; } = -1; + public int MaxOutStandingMessages { get; set; } = -1; /// /// At what interval should we check the number of outstanding messages has not exceeded the limit set in MaxOutStandingMessages diff --git a/src/Paramore.Brighter/RelationDatabaseOutbox.cs b/src/Paramore.Brighter/RelationDatabaseOutbox.cs index 08deb53010..c21911eb7f 100644 --- a/src/Paramore.Brighter/RelationDatabaseOutbox.cs +++ b/src/Paramore.Brighter/RelationDatabaseOutbox.cs @@ -918,7 +918,7 @@ public IEnumerable OutstandingMessages( /// Outstanding Messages public async Task> OutstandingMessagesAsync( TimeSpan dispatchedSince, - RequestContext requestContext, + RequestContext? requestContext, int pageSize = 100, int pageNumber = 1, IEnumerable? trippedTopics = null, @@ -951,6 +951,19 @@ public async Task> OutstandingMessagesAsync( } } + /// + public int GetOutstandingMessageCount(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null) + { + return OutstandingMessages(dispatchedSince, requestContext, maxCount, 1, null, args).Count(); + } + + /// + public async Task GetOutstandingMessageCountAsync(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null, CancellationToken cancellationToken = default) + { + return (await OutstandingMessagesAsync(dispatchedSince, requestContext, maxCount, 1, null, args, cancellationToken)) + .Count(); + } + protected virtual void WriteToStore( IAmABoxTransactionProvider? transactionProvider, Func commandFunc, diff --git a/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs b/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs index 355fd35313..f3b08dc40e 100644 --- a/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs +++ b/tests/Paramore.Brighter.Core.Tests/CommandProcessors/TestDoubles/SpyOutbox.cs @@ -122,6 +122,16 @@ public Task> OutstandingMessagesAsync(TimeSpan dispatchedSi { return Task.FromResult(OutstandingMessages(dispatchedSince, requestContext, pageSize, pageNumber)); } + + public int GetOutstandingMessageCount(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null) + { + return OutstandingMessages(dispatchedSince, requestContext, maxCount, 1, null, args).Count(); + } + + public Task GetOutstandingMessageCountAsync(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null, CancellationToken cancellationToken = default) + { + return Task.FromResult(GetOutstandingMessageCount(dispatchedSince, requestContext, maxCount, args)); + } } public class SpyOutboxEntry From 38b300bf3bd212f9a1f3b43b808f8058a368bf06 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Tue, 23 Sep 2025 13:24:40 +0100 Subject: [PATCH 13/22] Make shard assignment deterministic This commit makes shard assignment deterministic based on the hash of the partition key for a given message, making it possible to guarantee message ordering when querying for outstanding messages for a particular topic. --- .../DynamoDbOutbox.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 026f086b8c..7eaac67292 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -27,6 +27,8 @@ THE SOFTWARE. */ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Security.Cryptography; +using System.Text; using System.Threading; using System.Threading.Tasks; using Amazon.DynamoDBv2; @@ -169,7 +171,7 @@ public async Task AddAsync( try { - var shard = GetShardNumber(); + var shard = GetShardNumber(message.Header.PartitionKey); var expiresAt = GetExpirationTime(); var messageToStore = new MessageItem(message, shard, expiresAt); @@ -1094,11 +1096,19 @@ await _context.SaveAsync( .ConfigureAwait(ContinueOnCapturedContext); } - private int GetShardNumber() + private int GetShardNumber(string? partitionKey) { if (_configuration.NumberOfShards <= 1) return 0; + if (partitionKey != null) + { + var keyBytes = Encoding.UTF8.GetBytes(partitionKey); + var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(keyBytes); + return BitConverter.ToInt32(hash, 0) % _configuration.NumberOfShards; + } + //The range is inclusive of 0 but exclusive of NumberOfShards i.e. 0, 4 produces values in range 0-3 return _random.Next(0, _configuration.NumberOfShards); } From ba44724d236d7d1a9922293084cc2b4245248cff Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Tue, 23 Sep 2025 13:25:46 +0100 Subject: [PATCH 14/22] Port Dynamo DB shard assignment to v4 sdk --- .../DynamoDbOutbox.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index 4033e29bdf..b0c4724515 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -27,6 +27,7 @@ THE SOFTWARE. */ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using Amazon.DynamoDBv2; @@ -182,7 +183,7 @@ public async Task AddAsync( try { - var shard = GetShardNumber(); + var shard = GetShardNumber(message.Header.PartitionKey); var expiresAt = GetExpirationTime(); var messageToStore = new MessageItem(message, shard, expiresAt); @@ -1107,11 +1108,19 @@ await _context.SaveAsync( .ConfigureAwait(ContinueOnCapturedContext); } - private int GetShardNumber() + private int GetShardNumber(string? partitionKey) { if (_configuration.NumberOfShards <= 1) return 0; + if (partitionKey != null) + { + var keyBytes = Encoding.UTF8.GetBytes(partitionKey); + var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(keyBytes); + return BitConverter.ToInt32(hash, 0) % _configuration.NumberOfShards; + } + //The range is inclusive of 0 but exclusive of NumberOfShards i.e. 0, 4 produces values in range 0-3 return _random.Next(0, _configuration.NumberOfShards); } From d28a7017d4eea1a71451e75eb2efd2a9a8116eda Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Tue, 23 Sep 2025 13:30:00 +0100 Subject: [PATCH 15/22] Remove misleading comment --- src/Paramore.Brighter/OutboxProducerMediator.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Paramore.Brighter/OutboxProducerMediator.cs b/src/Paramore.Brighter/OutboxProducerMediator.cs index 158b335d7b..95d7142565 100644 --- a/src/Paramore.Brighter/OutboxProducerMediator.cs +++ b/src/Paramore.Brighter/OutboxProducerMediator.cs @@ -74,7 +74,6 @@ public partial class OutboxProducerMediator : IAmAnOutbo private const string NoSyncOutboxError = "A sync Outbox must be defined."; private const string NoAsyncOutboxError = "An async Outbox must be defined."; - //Uses -1 to indicate no outbox and will thus force a throw on a failed publish private int _outStandingCount; private bool _disposed; private readonly int _maxOutStandingMessages; From 3c2f57234fd0549794a4b5caf4f6d2dfa8e70fcf Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Thu, 25 Sep 2025 14:55:58 +0100 Subject: [PATCH 16/22] Fixes for existing tests This fixes all the issues highlighted by the pre-existing dynamo DB outbox tests --- .../DynamoDbOutbox.cs | 18 ++++---------- .../MessageItem.cs | 8 ++++--- .../DynamoDbOutbox.cs | 18 ++++---------- .../Outbox/DynamoDBOutboxBaseTest.cs | 4 +++- ...e_are_dispatched_messages_in_the_outbox.cs | 20 ++++++++-------- ..._are_outstanding_messages_in_the_outbox.cs | 20 +++++++--------- .../Outbox/DynamoDBOutboxBaseTest.cs | 6 +++-- ...e_are_dispatched_messages_in_the_outbox.cs | 24 +++++++++---------- ..._are_outstanding_messages_in_the_outbox.cs | 24 +++++++++---------- 9 files changed, 64 insertions(+), 78 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index b0c4724515..468ddb6e37 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -755,16 +755,12 @@ private async Task> ScanOutstandingIndexSegmentForMessages(Dat int segmentNumber, CancellationToken cancellationToken) { - string? paginationToken = null; + string? paginationToken = _outstandingAllTopicsScanContext.GetPagingToken(segmentNumber); if (pageNumber != 1 && paginationToken == null) { // It may be that this segment is done but other segments have more results return new List(); } - else if (pageNumber != 1) - { - paginationToken = _outstandingAllTopicsScanContext.GetPagingToken(segmentNumber); - } var segmentMessages = new List(); do @@ -793,7 +789,7 @@ private async Task> ScanOutstandingIndexSegmentForMessages(Dat segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); - paginationToken = scan.PaginationToken; + paginationToken = scan.IsDone ? null : scan.PaginationToken; } while (paginationToken != null && segmentMessages.Count < pageSize); // If there are more results, store the context for retrieving the next page @@ -917,7 +913,7 @@ private async Task> DispatchedMessagesForTopicAsync( // Store the progress for this topic if there are further pages if (!queryResult.QueryComplete) { - context.SetPaginationState(pageNumber + 1, 0, queryResult.PaginationToken); + context.SetPaginationState(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken); } else { @@ -982,16 +978,12 @@ private async Task> ScanDispatchedIndexSegment(DateTimeOffset int segmentNumber, CancellationToken cancellationToken) { - string? paginationToken = null; + string? paginationToken = _dispatchedAllTopicsScanContext.GetPagingToken(segmentNumber); if (pageNumber != 1 && paginationToken == null) { // It may be that this segment is done but other segments have more results return new List(); } - else if (pageNumber != 1) - { - paginationToken = _dispatchedAllTopicsScanContext.GetPagingToken(segmentNumber); - } var segmentMessages = new List(); do @@ -1020,7 +1012,7 @@ private async Task> ScanDispatchedIndexSegment(DateTimeOffset segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); - paginationToken = scan.PaginationToken; + paginationToken = scan.IsDone ? null : scan.PaginationToken; } while (paginationToken != null && segmentMessages.Count < pageSize); // If there are more results, store the context for retrieving the next page diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/MessageItem.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/MessageItem.cs index 7b2115ffce..34b25c13bf 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/MessageItem.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/MessageItem.cs @@ -58,6 +58,7 @@ public class MessageItem /// /// The outstanding creation time as a in ticks. May be null. [DynamoDBGlobalSecondaryIndexRangeKey(indexName: "Outstanding")] + [DynamoDBGlobalSecondaryIndexHashKey(indexName: "OutstandingAllTopics")] [DynamoDBProperty] public long? OutstandingCreatedTime { get; set; } @@ -91,6 +92,7 @@ public class MessageItem /// /// The delivery time as a in ticks. May be null. [DynamoDBGlobalSecondaryIndexRangeKey(indexName: "Delivered")] + [DynamoDBGlobalSecondaryIndexHashKey(indexName: "DeliveredAllTopics")] [DynamoDBProperty] public long? DeliveryTime { get; set; } @@ -115,6 +117,7 @@ public class MessageItem /// /// The message ID as a . May be null. [DynamoDBHashKey] + [DynamoDBGlobalSecondaryIndexRangeKey(indexNames: ["OutstandingAllTopics", "DeliveredAllTopics"])] [DynamoDBProperty] public string? MessageId { get; set; } @@ -160,15 +163,14 @@ public class MessageItem /// The Topic the message was published to /// /// The topic as a . May be null. - [DynamoDBGlobalSecondaryIndexHashKey("Delivered")] [DynamoDBProperty] public string? Topic { get; set; } - + /// /// The Topic suffixed with the shard number /// /// The topic shard as a . May be null. - [DynamoDBGlobalSecondaryIndexHashKey("Outstanding")] + [DynamoDBGlobalSecondaryIndexHashKey(["Outstanding", "Delivered"])] [DynamoDBProperty] public string? TopicShard { get; set; } diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 7eaac67292..c2438cd7af 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -743,16 +743,12 @@ private async Task> ScanOutstandingIndexSegmentForMessages(Dat int segmentNumber, CancellationToken cancellationToken) { - string? paginationToken = null; + var paginationToken = _outstandingAllTopicsScanContext.GetPagingToken(segmentNumber); if (pageNumber != 1 && paginationToken == null) { // It may be that this segment is done but other segments have more results return new List(); } - else if (pageNumber != 1) - { - paginationToken = _outstandingAllTopicsScanContext.GetPagingToken(segmentNumber); - } var segmentMessages = new List(); do @@ -781,7 +777,7 @@ private async Task> ScanOutstandingIndexSegmentForMessages(Dat segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); - paginationToken = scan.PaginationToken; + paginationToken = scan.IsDone ? null : scan.PaginationToken; } while (paginationToken != null && segmentMessages.Count < pageSize); // If there are more results, store the context for retrieving the next page @@ -905,7 +901,7 @@ private async Task> DispatchedMessagesForTopicAsync( // Store the progress for this topic if there are further pages if (!queryResult.QueryComplete) { - context.SetPaginationState(pageNumber + 1, 0, queryResult.PaginationToken); + context.SetPaginationState(pageNumber + 1, queryResult.ShardNumber, queryResult.PaginationToken); } else { @@ -970,16 +966,12 @@ private async Task> ScanDispatchedIndexSegment(DateTimeOffset int segmentNumber, CancellationToken cancellationToken) { - string? paginationToken = null; + string? paginationToken = _dispatchedAllTopicsScanContext.GetPagingToken(segmentNumber); if (pageNumber != 1 && paginationToken == null) { // It may be that this segment is done but other segments have more results return new List(); } - else if (pageNumber != 1) - { - paginationToken = _dispatchedAllTopicsScanContext.GetPagingToken(segmentNumber); - } var segmentMessages = new List(); do @@ -1008,7 +1000,7 @@ private async Task> ScanDispatchedIndexSegment(DateTimeOffset segmentMessages.AddRange(await scan.GetNextSetAsync(cancellationToken)); - paginationToken = scan.PaginationToken; + paginationToken = scan.IsDone ? null : scan.PaginationToken; } while (paginationToken != null && segmentMessages.Count < pageSize); // If there are more results, store the context for retrieving the next page diff --git a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/DynamoDBOutboxBaseTest.cs b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/DynamoDBOutboxBaseTest.cs index 1293af1d7b..458faa515f 100644 --- a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/DynamoDBOutboxBaseTest.cs +++ b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/DynamoDBOutboxBaseTest.cs @@ -29,7 +29,9 @@ protected DynamoDBOutboxBaseTest () new Dictionary { {"Outstanding", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}}, - {"Delivered", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}} + {"OutstandingAllTopics", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}}, + {"Delivered", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}}, + {"DeliveredAllTopics", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}} } )); OutboxTableName = createTableRequest.TableName; diff --git a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs index b41861f82d..aa807f75f2 100644 --- a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs +++ b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs @@ -33,7 +33,7 @@ public async Task When_there_are_dispatched_messages_in_the_outbox_async() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 1, args: args); + var messages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -52,7 +52,7 @@ public async Task When_there_are_dispatched_messages_in_the_outbox() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 1, args: args); + var messages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -76,7 +76,7 @@ public async Task When_there_are_dispatched_messages_for_multiple_topics_async() _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var dispatchedMessages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 1); + var dispatchedMessages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -89,7 +89,7 @@ public async Task When_there_are_dispatched_messages_for_multiple_topics_async() } [Fact] - public async Task When_there_are_dispatched_messages_for_multiple_topics() + public void When_there_are_dispatched_messages_for_multiple_topics() { var messages = new List(); var context = new RequestContext(); @@ -104,7 +104,7 @@ public async Task When_there_are_dispatched_messages_for_multiple_topics() _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var dispatchedMessages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 1); + var dispatchedMessages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -141,7 +141,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_a_top var dispatchedMessages = (await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 5, 1, args: args)).ToList(); Assert.Equal(5, dispatchedMessages.Count); // Get the remainder - dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 2, args: args)); + dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -177,7 +177,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_a_top var dispatchedMessages = (_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 5, 1, args: args)).ToList(); Assert.Equal(5, dispatchedMessages.Count); // Get the remainder - dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 2, args: args)); + dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -220,7 +220,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multi dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 5)); + dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -233,7 +233,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multi } [Fact] - public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multiple_topics() + public void When_there_are_multiple_pages_of_dispatched_messages_for_multiple_topics() { var context = new RequestContext(); var messages = new List(); @@ -264,7 +264,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multi dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 5)); + dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) diff --git a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs index f25124c9f0..c61620ac5b 100644 --- a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs +++ b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs @@ -32,7 +32,7 @@ public async Task When_there_are_outstanding_messages_in_the_outbox_async() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 1, args: args); + var messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -50,7 +50,7 @@ public void When_there_are_outstanding_messages_in_the_outbox() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages =_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 1, args: args); + var messages =_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -73,7 +73,7 @@ public async Task When_there_are_outstanding_messages_for_multiple_topics_async( _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var outstandingMessages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 1); + var outstandingMessages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -100,7 +100,7 @@ public void When_there_are_outstanding_messages_for_multiple_topics() _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var outstandingMessages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 1); + var outstandingMessages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -136,7 +136,7 @@ public async Task When_there_are_multiple_pages_of_outstanding_messages_for_a_to var outstandingMessages = (await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 5, 1, args: args)).ToList(); Assert.Equal(5, outstandingMessages.Count); // Get the remainder - outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 2, args: args)); + outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -171,7 +171,7 @@ public void When_there_are_multiple_pages_of_outstanding_messages_for_a_topic() var outstandingMessages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 5, 1, args: args).ToList(); Assert.Equal(5, outstandingMessages.Count); // Get the remainder - outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 2, args: args)); + outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -188,8 +188,6 @@ public async Task When_there_are_multiple_pages_of_outstanding_messages_for_all_ var context = new RequestContext(); var messages = new List(); - // Create enough messages to guarantee they will be split across multiple shards - // for all topics var topics = new[] { "one_topic", "another_topic" }; foreach (var topic in topics) { @@ -213,7 +211,7 @@ public async Task When_there_are_multiple_pages_of_outstanding_messages_for_all_ outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 5)); + outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -231,8 +229,6 @@ public void When_there_are_multiple_pages_of_outstanding_messages_for_all_topics var context = new RequestContext(); var messages = new List(); - // Create enough messages to guarantee they will be split across multiple shards - // for all topics var topics = new[] { "one_topic", "another_topic" }; foreach (var topic in topics) { @@ -256,7 +252,7 @@ public void When_there_are_multiple_pages_of_outstanding_messages_for_all_topics outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 5)); + outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) diff --git a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/DynamoDBOutboxBaseTest.cs b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/DynamoDBOutboxBaseTest.cs index d102015952..72e1aab4f1 100644 --- a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/DynamoDBOutboxBaseTest.cs +++ b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/DynamoDBOutboxBaseTest.cs @@ -31,7 +31,9 @@ protected DynamoDBOutboxBaseTest () new Dictionary { {"Outstanding", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}}, - {"Delivered", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}} + {"OutstandingAllTopics", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}}, + {"Delivered", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}}, + {"DeliveredAllTopics", new ProvisionedThroughput{ReadCapacityUnits = 10, WriteCapacityUnits = 10}} } )); OutboxTableName = createTableRequest.TableName; @@ -83,4 +85,4 @@ protected virtual void Dispose(bool disposing) _disposed = true; } -} \ No newline at end of file +} diff --git a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs index 524df23742..4623d88fa2 100644 --- a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs +++ b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_dispatched_messages_in_the_outbox.cs @@ -33,7 +33,7 @@ public async Task When_there_are_dispatched_messages_in_the_outbox_async() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 1, args: args); + var messages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -42,7 +42,7 @@ public async Task When_there_are_dispatched_messages_in_the_outbox_async() } [Fact] - public async Task When_there_are_dispatched_messages_in_the_outbox() + public void When_there_are_dispatched_messages_in_the_outbox() { var context = new RequestContext(); _dynamoDbOutbox.Add(_message, context); @@ -52,7 +52,7 @@ public async Task When_there_are_dispatched_messages_in_the_outbox() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 1, args: args); + var messages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -76,7 +76,7 @@ public async Task When_there_are_dispatched_messages_for_multiple_topics_async() _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var dispatchedMessages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 1); + var dispatchedMessages = await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -89,7 +89,7 @@ public async Task When_there_are_dispatched_messages_for_multiple_topics_async() } [Fact] - public async Task When_there_are_dispatched_messages_for_multiple_topics() + public void When_there_are_dispatched_messages_for_multiple_topics() { var messages = new List(); var context = new RequestContext(); @@ -104,7 +104,7 @@ public async Task When_there_are_dispatched_messages_for_multiple_topics() _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var dispatchedMessages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 1); + var dispatchedMessages = _dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -141,7 +141,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_a_top var dispatchedMessages = (await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 5, 1, args: args)).ToList(); Assert.Equal(5, dispatchedMessages.Count); // Get the remainder - dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 2, args: args)); + dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -153,7 +153,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_a_top } [Fact] - public async Task When_there_are_multiple_pages_of_dispatched_messages_for_a_topic() + public void When_there_are_multiple_pages_of_dispatched_messages_for_a_topic() { var context = new RequestContext(); var messages = new List(); @@ -177,7 +177,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_a_top var dispatchedMessages = (_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 5, 1, args: args)).ToList(); Assert.Equal(5, dispatchedMessages.Count); // Get the remainder - dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 2, args: args)); + dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -220,7 +220,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multi dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 100, 5)); + dispatchedMessages.AddRange(await _dynamoDbOutbox.DispatchedMessagesAsync(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -233,7 +233,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multi } [Fact] - public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multiple_topics() + public void When_there_are_multiple_pages_of_dispatched_messages_for_multiple_topics() { var context = new RequestContext(); var messages = new List(); @@ -264,7 +264,7 @@ public async Task When_there_are_multiple_pages_of_dispatched_messages_for_multi dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 100, 5)); + dispatchedMessages.AddRange(_dynamoDbOutbox.DispatchedMessages(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) diff --git a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs index 8d9cc390f2..09c9ffd19b 100644 --- a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs +++ b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_there_are_outstanding_messages_in_the_outbox.cs @@ -32,7 +32,7 @@ public async Task When_there_are_outstanding_messages_in_the_outbox_async() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 1, args: args); + var messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -50,7 +50,7 @@ public void When_there_are_outstanding_messages_in_the_outbox() var args = new Dictionary {{"Topic", "test_topic"}}; - var messages =_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 1, args: args); + var messages =_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -73,7 +73,7 @@ public async Task When_there_are_outstanding_messages_for_multiple_topics_async( _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var outstandingMessages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 1); + var outstandingMessages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -100,7 +100,7 @@ public void When_there_are_outstanding_messages_for_multiple_topics() _fakeTimeProvider.Advance(TimeSpan.FromSeconds(1)); - var outstandingMessages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 1); + var outstandingMessages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 1); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -136,7 +136,7 @@ public async Task When_there_are_multiple_pages_of_outstanding_messages_for_a_to var outstandingMessages = (await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 5, 1, args: args)).ToList(); Assert.Equal(5, outstandingMessages.Count); // Get the remainder - outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 2, args: args)); + outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -171,7 +171,7 @@ public void When_there_are_multiple_pages_of_outstanding_messages_for_a_topic() var outstandingMessages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 5, 1, args: args).ToList(); Assert.Equal(5, outstandingMessages.Count); // Get the remainder - outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 2, args: args)); + outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 2, args: args)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -213,7 +213,7 @@ public async Task When_there_are_multiple_pages_of_outstanding_messages_for_all_ outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 5)); + outstandingMessages.AddRange(await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -256,7 +256,7 @@ public void When_there_are_multiple_pages_of_outstanding_messages_for_all_topics outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 5, i)); } // Do a last page in case other tests have added more messages - outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 5)); + outstandingMessages.AddRange(_dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 5)); //Other tests may leave messages, so make sure that we grab ours foreach (var message in messages) @@ -278,7 +278,7 @@ public async Task When_an_outstanding_message_is_dispatched_async() var args = new Dictionary { { "Topic", "test_topic" } }; - var messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 1, args: args); + var messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -289,7 +289,7 @@ public async Task When_an_outstanding_message_is_dispatched_async() // Give the GSI a second to catch up await Task.Delay(1000); - messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 100, 1, args: args); + messages = await _dynamoDbOutbox.OutstandingMessagesAsync(TimeSpan.Zero, context, 1000, 1, args: args); messages.All(m => m.Id != _message.Id); } @@ -303,7 +303,7 @@ public async Task When_an_outstanding_message_is_dispatched() var args = new Dictionary { { "Topic", "test_topic" } }; - var messages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 1, args: args); + var messages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 1, args: args); //Other tests may leave messages, so make sure that we grab ours var message = messages.Single(m => m.Id == _message.Id); @@ -314,7 +314,7 @@ public async Task When_an_outstanding_message_is_dispatched() // Give the GSI a second to catch up await Task.Delay(1000); - messages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 100, 1, args: args); + messages = _dynamoDbOutbox.OutstandingMessages(TimeSpan.Zero, context, 1000, 1, args: args); messages.All(m => m.Id != _message.Id); } From 96d396025001f719b7eec50d7c89a7a76bb2984a Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Thu, 25 Sep 2025 15:19:08 +0100 Subject: [PATCH 17/22] Ensure accurate segment page sizes If the user has configured a scan concurrency that doesn't neatly divide into the requested page size, then the total number of scanned records will be less than the requested number. This commit ensures that the remainder is divided between the segments correctly. --- .../DynamoDbOutbox.cs | 22 +++++++++++++++---- .../DynamoDbOutbox.cs | 21 ++++++++++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index 468ddb6e37..63fc445f6d 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -27,6 +27,7 @@ THE SOFTWARE. */ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -727,10 +728,10 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti // Spin off requests to scan each segment var tasks = new List>>(); - var segmentPageSize = pageSize / _configuration.ScanConcurrency; + var segmentPageSizes = GetSegmentPageSizes(pageSize); for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanOutstandingIndexSegmentForMessages(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + tasks.Add(ScanOutstandingIndexSegmentForMessages(olderThan, segmentPageSizes[segmentNumber], pageNumber, segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); @@ -749,6 +750,19 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti } } + private int[] GetSegmentPageSizes(int pageSize) + { + if (pageSize % _configuration.ScanConcurrency == 0) + { + return Enumerable.Repeat(pageSize / _configuration.ScanConcurrency, _configuration.ScanConcurrency).ToArray(); + } + + var remainder = pageSize % _configuration.ScanConcurrency; + var segmentPageSizes = Enumerable.Repeat((pageSize / _configuration.ScanConcurrency) + 1, remainder).ToList(); + segmentPageSizes.AddRange(Enumerable.Repeat(pageSize / _configuration.ScanConcurrency, _configuration.ScanConcurrency - remainder)); + return segmentPageSizes.ToArray(); + } + private async Task> ScanOutstandingIndexSegmentForMessages(DateTimeOffset olderThan, int pageSize, int pageNumber, @@ -950,10 +964,10 @@ private async Task> DispatchedMessagesForAllTopicsAsync( // Spin off requests to scan each segment var tasks = new List>>(); - var segmentPageSize = pageSize / _configuration.ScanConcurrency; + var segmentPageSizes = GetSegmentPageSizes(pageSize); for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanDispatchedIndexSegment(dispatchedBefore, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + tasks.Add(ScanDispatchedIndexSegment(dispatchedBefore, segmentPageSizes[segmentNumber], pageNumber, segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index c2438cd7af..815964b529 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -715,10 +715,10 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti // Spin off requests to scan each segment var tasks = new List>>(); - var segmentPageSize = pageSize / _configuration.ScanConcurrency; + var segmentPageSizes = GetSegmentPageSizes(pageSize); for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanOutstandingIndexSegmentForMessages(olderThan, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + tasks.Add(ScanOutstandingIndexSegmentForMessages(olderThan, segmentPageSizes[segmentNumber], pageNumber, segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); @@ -737,6 +737,19 @@ private async Task> OutstandingMessagesForAllTopicsAsync(Ti } } + private int[] GetSegmentPageSizes(int pageSize) + { + if (pageSize % _configuration.ScanConcurrency == 0) + { + return Enumerable.Repeat(pageSize / _configuration.ScanConcurrency, _configuration.ScanConcurrency).ToArray(); + } + + var remainder = pageSize % _configuration.ScanConcurrency; + var segmentPageSizes = Enumerable.Repeat((pageSize / _configuration.ScanConcurrency) + 1, remainder).ToList(); + segmentPageSizes.AddRange(Enumerable.Repeat(pageSize / _configuration.ScanConcurrency, _configuration.ScanConcurrency - remainder)); + return segmentPageSizes.ToArray(); + } + private async Task> ScanOutstandingIndexSegmentForMessages(DateTimeOffset olderThan, int pageSize, int pageNumber, @@ -938,10 +951,10 @@ private async Task> DispatchedMessagesForAllTopicsAsync( // Spin off requests to scan each segment var tasks = new List>>(); - var segmentPageSize = pageSize / _configuration.ScanConcurrency; + var segmentPageSizes = GetSegmentPageSizes(pageSize); for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanDispatchedIndexSegment(dispatchedBefore, segmentPageSize, pageNumber, segmentNumber, cancellationToken)); + tasks.Add(ScanDispatchedIndexSegment(dispatchedBefore, segmentPageSizes[segmentNumber], pageNumber, segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); From f474ebff4131c866b883ebc54bb25c191c6f24c3 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Thu, 25 Sep 2025 16:05:51 +0100 Subject: [PATCH 18/22] Add new tests for new outbox methods This commit adds new tests to cover the 4 new outbox methods for the Dynamo DB implementation, namely: - Getting a batch of messages by ID - Counting the number of outstanding messages --- .../DynamoDbOutbox.cs | 6 +- .../DynamoDbOutbox.cs | 6 +- .../When_counting_outstanding_messages.cs | 82 +++++++++++++++++++ ...ing_a_batch_of_messages_from_the_outbox.cs | 76 +++++++++++++++++ .../When_counting_outstanding_messages.cs | 82 +++++++++++++++++++ ...ing_a_batch_of_messages_from_the_outbox.cs | 76 +++++++++++++++++ 6 files changed, 322 insertions(+), 6 deletions(-) create mode 100644 tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_counting_outstanding_messages.cs create mode 100644 tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs create mode 100644 tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_counting_outstanding_messages.cs create mode 100644 tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs diff --git a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs index 63fc445f6d..fb1e1b11ec 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB.V4/DynamoDbOutbox.cs @@ -658,10 +658,10 @@ public async Task GetOutstandingMessageCountAsync( // Spin off requests to scan each segment var tasks = new List>(); - var segmentMaxCount = maxCount / _configuration.ScanConcurrency; + var segmentMaxCounts = GetSegmentPageSizes(maxCount); for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanOutstandingIndexSegmentForCount(olderThan, segmentMaxCount, segmentNumber, cancellationToken)); + tasks.Add(ScanOutstandingIndexSegmentForCount(olderThan, segmentMaxCounts[segmentNumber], segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); @@ -689,7 +689,7 @@ private async Task ScanOutstandingIndexSegmentForCount(DateTimeOffset older { TableName = _configuration.TableName, IndexName = _configuration.OutstandingAllTopicsIndexName, - ConsistentRead = true, + ConsistentRead = false, Limit = maxCount - segmentCount, ExclusiveStartKey = lastEvaluatedKey, Segment = segmentNumber, diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 815964b529..919351ab6e 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -645,10 +645,10 @@ public async Task GetOutstandingMessageCountAsync( // Spin off requests to scan each segment var tasks = new List>(); - var segmentMaxCount = maxCount / _configuration.ScanConcurrency; + var segmentMaxCounts = GetSegmentPageSizes(maxCount); for (var segmentNumber = 0; segmentNumber < _configuration.ScanConcurrency; segmentNumber++) { - tasks.Add(ScanOutstandingIndexSegmentForCount(olderThan, segmentMaxCount, segmentNumber, cancellationToken)); + tasks.Add(ScanOutstandingIndexSegmentForCount(olderThan, segmentMaxCounts[segmentNumber], segmentNumber, cancellationToken)); } await Task.WhenAll(tasks); @@ -676,7 +676,7 @@ private async Task ScanOutstandingIndexSegmentForCount(DateTimeOffset older { TableName = _configuration.TableName, IndexName = _configuration.OutstandingAllTopicsIndexName, - ConsistentRead = true, + ConsistentRead = false, Limit = maxCount - segmentCount, ExclusiveStartKey = lastEvaluatedKey, Segment = segmentNumber, diff --git a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_counting_outstanding_messages.cs b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_counting_outstanding_messages.cs new file mode 100644 index 0000000000..ae96b50e30 --- /dev/null +++ b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_counting_outstanding_messages.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Outbox.DynamoDB; +using Xunit; + +namespace Paramore.Brighter.DynamoDB.Tests.Outbox; + +[Trait("Category", "DynamoDB")] +public class DynamoDbOutboxOutstandingMessageCountTests : DynamoDBOutboxBaseTest +{ + private readonly DynamoDbOutbox _dynamoDbOutbox; + private readonly FakeTimeProvider _fakeTimeProvider; + + public DynamoDbOutboxOutstandingMessageCountTests() + { + // Set the fake time provider to an early time to prevent conflicts from other tests + _fakeTimeProvider = new FakeTimeProvider(new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero)); + _dynamoDbOutbox = new DynamoDbOutbox(Client, new DynamoDbConfiguration(OutboxTableName), _fakeTimeProvider); + } + + [Fact] + public async Task When_counting_outstanding_messages() + { + var context = new RequestContext(); + var initialOutstandingCount = _dynamoDbOutbox.GetOutstandingMessageCount(TimeSpan.Zero, context); + + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + _dynamoDbOutbox.Add(message, context); + } + + // Mark a couple as dispatched + _dynamoDbOutbox.MarkDispatched(messages[0].Id, context); + _dynamoDbOutbox.MarkDispatched(messages[1].Id, context); + + // Give the GSI a second to catch up + await Task.Delay(1000); + _fakeTimeProvider.Advance(TimeSpan.FromMinutes(1)); + + var finalOutstandingCount = _dynamoDbOutbox.GetOutstandingMessageCount(TimeSpan.Zero, context); + Assert.Equal(initialOutstandingCount + 8, finalOutstandingCount); + } + + [Fact] + public async Task When_counting_outstanding_messages_async() + { + var context = new RequestContext(); + var initialOutstandingCount = await _dynamoDbOutbox.GetOutstandingMessageCountAsync(TimeSpan.Zero, context); + + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + await _dynamoDbOutbox.AddAsync(message, context); + } + + // Mark a couple as dispatched + await _dynamoDbOutbox.MarkDispatchedAsync(messages[0].Id, context); + await _dynamoDbOutbox.MarkDispatchedAsync(messages[1].Id, context); + + // Give the GSI a second to catch up + await Task.Delay(1000); + _fakeTimeProvider.Advance(TimeSpan.FromMinutes(1)); + + var finalOutstandingCount = await _dynamoDbOutbox.GetOutstandingMessageCountAsync(TimeSpan.Zero, context); + Assert.Equal(initialOutstandingCount + 8, finalOutstandingCount); + } + + private Message CreateMessage(string topic) + { + return new Message( + new MessageHeader(Guid.NewGuid().ToString(), new RoutingKey(topic), MessageType.MT_DOCUMENT, timeStamp: _fakeTimeProvider.GetUtcNow()), + new MessageBody("message body") + ); + } +} diff --git a/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs new file mode 100644 index 0000000000..ef993fb023 --- /dev/null +++ b/tests/Paramore.Brighter.DynamoDB.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Outbox.DynamoDB; +using Xunit; + +namespace Paramore.Brighter.DynamoDB.Tests.Outbox; + +[Trait("Category", "DynamoDB")] +public class DynamoDbOutboxBatchGetTests : DynamoDBOutboxBaseTest +{ + private readonly DynamoDbOutbox _dynamoDbOutbox; + private readonly FakeTimeProvider _fakeTimeProvider; + + public DynamoDbOutboxBatchGetTests() + { + _fakeTimeProvider = new FakeTimeProvider(); + _dynamoDbOutbox = new DynamoDbOutbox(Client, new DynamoDbConfiguration(OutboxTableName), _fakeTimeProvider); + } + + [Fact] + public void When_getting_a_batch_of_messages_from_the_outbox() + { + var context = new RequestContext(); + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + _dynamoDbOutbox.Add(message, context); + } + + var messageIds = messages.Select(m => m.Id); + var messagesFromOutbox = _dynamoDbOutbox.Get(messageIds, context); + + foreach (var message in messages) + { + var messageFromOutbox = messagesFromOutbox.SingleOrDefault(m => m.Id == message.Id); + Assert.NotNull(messageFromOutbox); + Assert.Equal(message.Body.Value, messageFromOutbox.Body.Value); + } + } + + [Fact] + public async Task When_getting_a_batch_of_messages_from_the_outbox_async() + { + var context = new RequestContext(); + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + await _dynamoDbOutbox.AddAsync(message, context); + } + + var messageIds = messages.Select(m => m.Id); + var messagesFromOutbox = await _dynamoDbOutbox.GetAsync(messageIds, context); + + foreach (var message in messages) + { + var messageFromOutbox = messagesFromOutbox.SingleOrDefault(m => m.Id == message.Id); + Assert.NotNull(messageFromOutbox); + Assert.Equal(message.Body.Value, messageFromOutbox.Body.Value); + } + } + + private Message CreateMessage(string topic) + { + return new Message( + new MessageHeader(Guid.NewGuid().ToString(), new RoutingKey(topic), MessageType.MT_DOCUMENT), + new MessageBody("message body") + ); + } +} diff --git a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_counting_outstanding_messages.cs b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_counting_outstanding_messages.cs new file mode 100644 index 0000000000..ba818ccb2f --- /dev/null +++ b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_counting_outstanding_messages.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Outbox.DynamoDB.V4; +using Xunit; + +namespace Paramore.Brighter.DynamoDB.V4.Tests.Outbox; + +[Trait("Category", "DynamoDB")] +public class DynamoDbOutboxOutstandingMessageCountTests : DynamoDBOutboxBaseTest +{ + private readonly DynamoDbOutbox _dynamoDbOutbox; + private readonly FakeTimeProvider _fakeTimeProvider; + + public DynamoDbOutboxOutstandingMessageCountTests() + { + // Set the fake time provider to an early time to prevent conflicts from other tests + _fakeTimeProvider = new FakeTimeProvider(new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero)); + _dynamoDbOutbox = new DynamoDbOutbox(Client, new DynamoDbConfiguration(OutboxTableName), _fakeTimeProvider); + } + + [Fact] + public async Task When_counting_outstanding_messages() + { + var context = new RequestContext(); + var initialOutstandingCount = _dynamoDbOutbox.GetOutstandingMessageCount(TimeSpan.Zero, context); + + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + _dynamoDbOutbox.Add(message, context); + } + + // Mark a couple as dispatched + _dynamoDbOutbox.MarkDispatched(messages[0].Id, context); + _dynamoDbOutbox.MarkDispatched(messages[1].Id, context); + + // Give the GSI a second to catch up + await Task.Delay(1000); + _fakeTimeProvider.Advance(TimeSpan.FromMinutes(1)); + + var finalOutstandingCount = _dynamoDbOutbox.GetOutstandingMessageCount(TimeSpan.Zero, context); + Assert.Equal(initialOutstandingCount + 8, finalOutstandingCount); + } + + [Fact] + public async Task When_counting_outstanding_messages_async() + { + var context = new RequestContext(); + var initialOutstandingCount = await _dynamoDbOutbox.GetOutstandingMessageCountAsync(TimeSpan.Zero, context); + + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + await _dynamoDbOutbox.AddAsync(message, context); + } + + // Mark a couple as dispatched + await _dynamoDbOutbox.MarkDispatchedAsync(messages[0].Id, context); + await _dynamoDbOutbox.MarkDispatchedAsync(messages[1].Id, context); + + // Give the GSI a second to catch up + await Task.Delay(1000); + _fakeTimeProvider.Advance(TimeSpan.FromMinutes(1)); + + var finalOutstandingCount = await _dynamoDbOutbox.GetOutstandingMessageCountAsync(TimeSpan.Zero, context); + Assert.Equal(initialOutstandingCount + 8, finalOutstandingCount); + } + + private Message CreateMessage(string topic) + { + return new Message( + new MessageHeader(Guid.NewGuid().ToString(), new RoutingKey(topic), MessageType.MT_DOCUMENT, timeStamp: _fakeTimeProvider.GetUtcNow()), + new MessageBody("message body") + ); + } +} diff --git a/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs new file mode 100644 index 0000000000..1f16b49454 --- /dev/null +++ b/tests/Paramore.Brighter.DynamoDB.V4.Tests/Outbox/When_getting_a_batch_of_messages_from_the_outbox.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Outbox.DynamoDB.V4; +using Xunit; + +namespace Paramore.Brighter.DynamoDB.V4.Tests.Outbox; + +[Trait("Category", "DynamoDB")] +public class DynamoDbOutboxBatchGetTests : DynamoDBOutboxBaseTest +{ + private readonly DynamoDbOutbox _dynamoDbOutbox; + private readonly FakeTimeProvider _fakeTimeProvider; + + public DynamoDbOutboxBatchGetTests() + { + _fakeTimeProvider = new FakeTimeProvider(); + _dynamoDbOutbox = new DynamoDbOutbox(Client, new DynamoDbConfiguration(OutboxTableName), _fakeTimeProvider); + } + + [Fact] + public void When_getting_a_batch_of_messages_from_the_outbox() + { + var context = new RequestContext(); + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + _dynamoDbOutbox.Add(message, context); + } + + var messageIds = messages.Select(m => m.Id); + var messagesFromOutbox = _dynamoDbOutbox.Get(messageIds, context); + + foreach (var message in messages) + { + var messageFromOutbox = messagesFromOutbox.SingleOrDefault(m => m.Id == message.Id); + Assert.NotNull(messageFromOutbox); + Assert.Equal(message.Body.Value, messageFromOutbox.Body.Value); + } + } + + [Fact] + public async Task When_getting_a_batch_of_messages_from_the_outbox_async() + { + var context = new RequestContext(); + var messages = new List(); + for (var i = 0; i < 10; i++) + { + var message = CreateMessage("test_topic"); + messages.Add(message); + await _dynamoDbOutbox.AddAsync(message, context); + } + + var messageIds = messages.Select(m => m.Id); + var messagesFromOutbox = await _dynamoDbOutbox.GetAsync(messageIds, context); + + foreach (var message in messages) + { + var messageFromOutbox = messagesFromOutbox.SingleOrDefault(m => m.Id == message.Id); + Assert.NotNull(messageFromOutbox); + Assert.Equal(message.Body.Value, messageFromOutbox.Body.Value); + } + } + + private Message CreateMessage(string topic) + { + return new Message( + new MessageHeader(Guid.NewGuid().ToString(), new RoutingKey(topic), MessageType.MT_DOCUMENT), + new MessageBody("message body") + ); + } +} From bea0163c9db262874247fec0cabc9e2087e7f300 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Fri, 26 Sep 2025 10:57:38 +0100 Subject: [PATCH 19/22] Add outbox migration guide to ADR This adds a migration guide to the ADR for these changes, for users who are currently using the Dynamo DB outbox with v9 and need to upgrade to v10. The upgrade can be done in place without needing a new outbox table. --- .../0033-optimise-reads-and-writes-from-dynamo-outbox.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md index 703e9f8a7e..f9960e965d 100644 --- a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md +++ b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md @@ -105,4 +105,11 @@ If the partition key isn't specified for a message, then fall back to random sha * When performing the `OutstandingMessages` or `DispatchedMessages` operations for all topics, we will only be able to guarantee the order of the returned messages if the number of outstanding messages is less than the page size for the operation. * Shards will be assigned to messages deterministically based on their partition key -* The possibility of future improvements to other outbox implementations, to take advantage of the new bulk operation methods \ No newline at end of file +* The possibility of future improvements to other outbox implementations, to take advantage of the new bulk operation methods +* Users of the Dynamo DB outbox implementation in Brighter v9 will need to update their table as part of their migration to v10: + * Add a new GSI called `OutstandingAllTopics`, that uses `OutstandingCreatedTime` as its HASH key and `MessageId` as its RANGE key + * Add a new GSI called `DeliveredAllTopics`, that uses `DeliveryTime` as its HASH key and `MessageId` as its RANGE key + * Change the HASH key used by the `Delivered` index. This can be achieved by: + * Adding a _new_ GSI, which for the sake of example we'll call `DeliveredV10`, which uses `TopicShard` as its HASH key and `DeliveryTime` as its RANGE key + * When performing the Brighter v10 upgrade, customise the `DynamoDbConfiguration` during configuration to set `DeliveredIndexName` to `DeliveredV10` + * Once the v10 upgrade is complete, the old `Delivered` index can be removed if desired \ No newline at end of file From 7fffa7f9fd81cbcbf4ab628d4dcbe9dbc996b09b Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Fri, 26 Sep 2025 13:29:32 +0100 Subject: [PATCH 20/22] Fix observability tests This fixes the observability tests, as they were looking for events in the wrong place (they're now on the per-operation create span instead of the per-message clear span as we pre-fetch all required messages in a batch). It also adds the missing span name for the new Count operation on outboxes. --- .../Observability/BrighterSpanExtensions.cs | 1 + ...n_Clearing_A_Message_A_Span_Is_Exported.cs | 60 +++++++++---------- ...ring_A_Message_A_Span_Is_Exported_Async.cs | 29 +++++---- ...ipile_Messages_Spans_Are_Exported_Async.cs | 6 +- ...ng_Multiple_Messages_Spans_Are_Exported.cs | 6 +- 5 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/Paramore.Brighter/Observability/BrighterSpanExtensions.cs b/src/Paramore.Brighter/Observability/BrighterSpanExtensions.cs index 54eb7999e0..c2a22df1f2 100644 --- a/src/Paramore.Brighter/Observability/BrighterSpanExtensions.cs +++ b/src/Paramore.Brighter/Observability/BrighterSpanExtensions.cs @@ -57,6 +57,7 @@ public static class BrighterSpanExtensions BoxDbOperation.Get => "retrieve.message", BoxDbOperation.MarkDispatched => "mark_as_dispatched.outstanding_messages", BoxDbOperation.OutStandingMessages => "retrieve.outstanding_messages", + BoxDbOperation.OutStandingMessageCount => "count.outstanding_messages", BoxDbOperation.Exists => "message.exists", _ => throw new ArgumentOutOfRangeException(nameof(span), span, null) }; diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported.cs b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported.cs index c6fa1453e3..a9da127b52 100644 --- a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported.cs +++ b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported.cs @@ -80,6 +80,36 @@ public void When_Clearing_A_Message_A_Span_Is_Exported(InstrumentationOptions op Assert.NotNull(createActivity); Assert.Equal(parentActivity?.Id, createActivity.ParentId); + var events = createActivity.Events.ToList(); + + //retrieving the message batch should be an event + var message = _internalBus.Stream(new RoutingKey("MyEvent")).Single(); + var getEvent = events.Single(e => e.Name == BoxDbOperation.Get.ToSpanName()); + + if (options == InstrumentationOptions.None) + Assert.Empty(getEvent.Tags); + + if (options.HasFlag(InstrumentationOptions.RequestBody)) + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageBody && a.Value as string == message.Body.Value); + if (options.HasFlag(InstrumentationOptions.Brighter)) + { + Assert.Contains(getEvent.Tags, a => a.Value != null && a.Key == BrighterSemanticConventions.OutboxSharedTransaction && (bool)a.Value == false); + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.OutboxType && a.Value as string == "sync"); + } + + if (options.HasFlag(InstrumentationOptions.Messaging)) + { + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageId && a.Value as string == message.Id.Value); + Assert.Contains(getEvent.Tags, + a => a.Key == BrighterSemanticConventions.MessagingDestination && a.Value?.ToString() == message.Header.Topic.Value); + Assert.Contains(getEvent.Tags, + a => a is { Value: not null, Key: BrighterSemanticConventions.MessageBodySize } && (int)a.Value == message.Body.Bytes.Length); + Assert.Contains(getEvent.Tags, + a => a.Key == BrighterSemanticConventions.MessageType && a.Value as string == message.Header.MessageType.ToString()); + Assert.Contains(getEvent.Tags, + a => a.Key == BrighterSemanticConventions.MessagingDestinationPartitionId && a.Value as string == message.Header.PartitionKey.Value); + } + //there should be a clear span for each message id var clearActivity = _exportedActivities.Single(a => a.DisplayName == $"{BrighterSemanticConventions.ClearMessages} {CommandProcessorSpanOperation.Clear.ToSpanName()}"); @@ -99,36 +129,6 @@ public void When_Clearing_A_Message_A_Span_Is_Exported(InstrumentationOptions op Assert.DoesNotContain(clearActivity.Tags, t => t.Key == BrighterSemanticConventions.MessageId); } - var events = clearActivity.Events.ToList(); - - //retrieving the message should be an event - var message = _internalBus.Stream(new RoutingKey("MyEvent")).Single(); - var depositEvent = events.Single(e => e.Name == BoxDbOperation.Get.ToSpanName()); - - if (options == InstrumentationOptions.None) - Assert.Empty(depositEvent.Tags); - - if (options.HasFlag(InstrumentationOptions.RequestBody)) - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageBody && a.Value as string == message.Body.Value); - if (options.HasFlag(InstrumentationOptions.Brighter)) - { - Assert.Contains(depositEvent.Tags, a => a.Value != null && a.Key == BrighterSemanticConventions.OutboxSharedTransaction && (bool)a.Value == false); - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.OutboxType && a.Value as string == "sync"); - } - - if (options.HasFlag(InstrumentationOptions.Messaging)) - { - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageId && a.Value as string == message.Id.Value); - Assert.Contains(depositEvent.Tags, - a => a.Key == BrighterSemanticConventions.MessagingDestination && a.Value?.ToString() == message.Header.Topic.Value); - Assert.Contains(depositEvent.Tags, - a => a is { Value: not null, Key: BrighterSemanticConventions.MessageBodySize } && (int)a.Value == message.Body.Bytes.Length); - Assert.Contains(depositEvent.Tags, - a => a.Key == BrighterSemanticConventions.MessageType && a.Value as string == message.Header.MessageType.ToString()); - Assert.Contains(depositEvent.Tags, - a => a.Key == BrighterSemanticConventions.MessagingDestinationPartitionId && a.Value as string == message.Header.PartitionKey.Value); - } - if (options.HasFlag(InstrumentationOptions.DatabaseInformation)) { //there should be a span in the Db for retrieving the message diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported_Async.cs b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported_Async.cs index 6eb0a3a4e1..00850b7253 100644 --- a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported_Async.cs +++ b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_A_Message_A_Span_Is_Exported_Async.cs @@ -130,26 +130,25 @@ public async Task When_Clearing_A_Message_A_Span_Is_Exported() Assert.Equal(parentActivity?.Id, createActivity.ParentId); Assert.Contains(createActivity.Tags, t => t is { Key: BrighterSemanticConventions.Operation, Value: "clear" }); - + var events = createActivity.Events.ToList(); + + //retrieving the message should be an event on the batch + var message = _internalBus.Stream(new RoutingKey(_topic)).Single(); + var getEvent = events.Single(e => e.Name == BoxDbOperation.Get.ToSpanName()); + Assert.Contains(getEvent.Tags, a => a.Value != null && a.Key == BrighterSemanticConventions.OutboxSharedTransaction && (bool)a.Value == false); + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.OutboxType && a.Value as string == "async"); + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageId && a.Value as string == message.Id.Value); + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.MessagingDestination && a.Value?.ToString() == message.Header.Topic.ToString()); + Assert.Contains(getEvent.Tags, a => a is { Value: not null, Key: BrighterSemanticConventions.MessageBodySize } && (int)a.Value == message.Body.Bytes.Length); + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageBody && a.Value as string == message.Body.Value); + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageType && a.Value as string == message.Header.MessageType.ToString()); + Assert.Contains(getEvent.Tags, a => a.Key == BrighterSemanticConventions.MessagingDestinationPartitionId && a.Value as string == message.Header.PartitionKey.Value); + //there should be a clear span for each message id var clearActivity = _exportedActivities.Single(a => a.DisplayName == $"{BrighterSemanticConventions.ClearMessages} {CommandProcessorSpanOperation.Clear.ToSpanName()}"); Assert.NotNull(clearActivity); Assert.Contains(clearActivity.Tags, t => t is { Key: BrighterSemanticConventions.Operation, Value: "clear" }); Assert.Contains(clearActivity.Tags, t => t.Key == BrighterSemanticConventions.MessageId && t.Value == messageId.Value); - - var events = clearActivity.Events.ToList(); - - //retrieving the message should be an event - var message = _internalBus.Stream(new RoutingKey(_topic)).Single(); - var depositEvent = events.Single(e => e.Name == BoxDbOperation.Get.ToSpanName()); - Assert.Contains(depositEvent.Tags, a => a.Value != null && a.Key == BrighterSemanticConventions.OutboxSharedTransaction && (bool)a.Value == false); - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.OutboxType && a.Value as string == "async"); - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageId && a.Value as string == message.Id.Value); - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.MessagingDestination && a.Value?.ToString() == message.Header.Topic.ToString()); - Assert.Contains(depositEvent.Tags, a => a is { Value: not null, Key: BrighterSemanticConventions.MessageBodySize } && (int)a.Value == message.Body.Bytes.Length); - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageBody && a.Value as string == message.Body.Value); - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.MessageType && a.Value as string == message.Header.MessageType.ToString()); - Assert.Contains(depositEvent.Tags, a => a.Key == BrighterSemanticConventions.MessagingDestinationPartitionId && a.Value as string == message.Header.PartitionKey.Value); //there should be a span in the Db for retrieving the message var outBoxActivity = _exportedActivities.Single(a => a.DisplayName == $"{BoxDbOperation.Get.ToSpanName()} {InMemoryAttributes.OutboxDbName} {InMemoryAttributes.DbTable}"); diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multipile_Messages_Spans_Are_Exported_Async.cs b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multipile_Messages_Spans_Are_Exported_Async.cs index 9ac49a53e7..79594b9fbd 100644 --- a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multipile_Messages_Spans_Are_Exported_Async.cs +++ b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multipile_Messages_Spans_Are_Exported_Async.cs @@ -127,7 +127,7 @@ public async Task When_Clearing_A_Message_A_Span_Is_Exported() _traceProvider.ForceFlush(); //assert - Assert.Equal(21, _exportedActivities.Count); + Assert.Equal(19, _exportedActivities.Count); Assert.Contains(_exportedActivities, a => a.Source.Name == "Paramore.Brighter"); //there should be a create span for the batch @@ -138,9 +138,9 @@ public async Task When_Clearing_A_Message_A_Span_Is_Exported() var clearActivity = _exportedActivities.Where(a => a.DisplayName == $"{BrighterSemanticConventions.ClearMessages} {CommandProcessorSpanOperation.Clear.ToSpanName()}"); Assert.Equal(3, clearActivity.Count()); - //there should be a span in the Db for retrieving the message + //there should be a span in the Db for retrieving the batch of messages var outBoxActivity = _exportedActivities.Where(a => a.DisplayName == $"{BoxDbOperation.Get.ToSpanName()} {InMemoryAttributes.OutboxDbName} {InMemoryAttributes.DbTable}"); - Assert.Equal(3, outBoxActivity.Count()); + Assert.Single(outBoxActivity); //there should be a span for publishing the message via the producer var producerActivity = _exportedActivities.Where(a => a.DisplayName == $"{_topic} {CommandProcessorSpanOperation.Publish.ToSpanName()}"); diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multiple_Messages_Spans_Are_Exported.cs b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multiple_Messages_Spans_Are_Exported.cs index 6733f45878..ae3181f418 100644 --- a/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multiple_Messages_Spans_Are_Exported.cs +++ b/tests/Paramore.Brighter.Core.Tests/Observability/CommandProcessor/Clear/When_Clearing_Multiple_Messages_Spans_Are_Exported.cs @@ -125,7 +125,7 @@ public void When_Clearing_A_Message_A_Span_Is_Exported() _traceProvider.ForceFlush(); //assert - Assert.Equal(21, _exportedActivities.Count); + Assert.Equal(19, _exportedActivities.Count); Assert.Contains(_exportedActivities, a => a.Source.Name == "Paramore.Brighter"); //there should be a create span for the batch @@ -136,9 +136,9 @@ public void When_Clearing_A_Message_A_Span_Is_Exported() var clearActivity = _exportedActivities.Where(a => a.DisplayName == $"{BrighterSemanticConventions.ClearMessages} {CommandProcessorSpanOperation.Clear.ToSpanName()}"); Assert.Equal(3, clearActivity.Count()); - //there should be a span in the Db for retrieving the message + //there should be a span in the Db for retrieving the batch of messages var outBoxActivity = _exportedActivities.Where(a => a.DisplayName == $"{BoxDbOperation.Get.ToSpanName()} {InMemoryAttributes.OutboxDbName} {InMemoryAttributes.DbTable}"); - Assert.Equal(3, outBoxActivity.Count()); + Assert.Single(outBoxActivity); //there should be a span for publishing the message via the producer var producerActivity = _exportedActivities.Where(a => a.DisplayName == $"{"MyEvent"} {CommandProcessorSpanOperation.Publish.ToSpanName()}"); From 67e7f5d1ef808c532d7b162c5af54dbc96dc8066 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Fri, 3 Oct 2025 13:46:53 +0100 Subject: [PATCH 21/22] Add clarifying comments This adds a couple of clarifying comments: - When calling the outstanding messages method on the Dynamo DB outbox, it makes it clear that only one query can run concurrently when querying all topics, and one query per topic can run when querying specific topics. - Updated the example index name in the guide for upgrading to the v10 outbox from a v9 outbox, to be clearer about what change is being made --- docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md | 4 ++-- src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md index f9960e965d..8272c89be6 100644 --- a/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md +++ b/docs/adr/0033-optimise-reads-and-writes-from-dynamo-outbox.md @@ -110,6 +110,6 @@ If the partition key isn't specified for a message, then fall back to random sha * Add a new GSI called `OutstandingAllTopics`, that uses `OutstandingCreatedTime` as its HASH key and `MessageId` as its RANGE key * Add a new GSI called `DeliveredAllTopics`, that uses `DeliveryTime` as its HASH key and `MessageId` as its RANGE key * Change the HASH key used by the `Delivered` index. This can be achieved by: - * Adding a _new_ GSI, which for the sake of example we'll call `DeliveredV10`, which uses `TopicShard` as its HASH key and `DeliveryTime` as its RANGE key - * When performing the Brighter v10 upgrade, customise the `DynamoDbConfiguration` during configuration to set `DeliveredIndexName` to `DeliveredV10` + * Adding a _new_ GSI, which for the sake of example we'll call `DeliveredSharded`, which uses `TopicShard` as its HASH key and `DeliveryTime` as its RANGE key + * When performing the Brighter v10 upgrade, customise the `DynamoDbConfiguration` during configuration to set `DeliveredIndexName` to `DeliveredSharded` * Once the v10 upgrade is complete, the old `Delivered` index can be removed if desired \ No newline at end of file diff --git a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs index 919351ab6e..6d57c93687 100644 --- a/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs +++ b/src/Paramore.Brighter.Outbox.DynamoDB/DynamoDbOutbox.cs @@ -568,7 +568,8 @@ public IEnumerable OutstandingMessages( } /// - /// Returns messages that have yet to be dispatched + /// Returns messages that have yet to be dispatched. When querying all topics, only one query can run concurrently. + /// When querying by topic, one query per topic can run concurrently. /// /// How long ago as the message sent? /// From d4727c9ada87dfa117496b01c54855cdc59b3a72 Mon Sep 17 00:00:00 2001 From: Dominic Hickie Date: Fri, 3 Oct 2025 14:24:49 +0100 Subject: [PATCH 22/22] Align GCP outbox with interface A new outbox implementation was added after this work was started - this commit aligns the new outbox with the changes made to the outbox interfaces. --- .../FirestoreOutbox.cs | 173 ++++++++---------- .../Outbox/When_retrieving_messages.cs | 3 +- .../Outbox/When_retrieving_messages_async.cs | 5 +- .../When_retrieving_outstanding_messages.cs | 4 +- ...n_retrieving_outstanding_messages_async.cs | 4 +- 5 files changed, 89 insertions(+), 100 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.Firestore/FirestoreOutbox.cs b/src/Paramore.Brighter.Outbox.Firestore/FirestoreOutbox.cs index 3d077f6057..3ebf69f2ce 100644 --- a/src/Paramore.Brighter.Outbox.Firestore/FirestoreOutbox.cs +++ b/src/Paramore.Brighter.Outbox.Firestore/FirestoreOutbox.cs @@ -319,6 +319,77 @@ public Message Get(Id messageId, RequestContext? requestContext, int outBoxTimeo } } + /// + public IEnumerable Get(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) + { + var ids = messageIds.Select(id => id.Value).ToArray(); + + var dbAttributes = new Dictionary() + { + {"db.operation.parameter.message.ids", string.Join(",", ids)} + }; + + var span = Tracer?.CreateDbSpan( + new BoxSpanInfo(DbSystem.Firestore, + _configuration.Database, + BoxDbOperation.Get, + _outboxCollection.Name, + dbAttributes: dbAttributes), + requestContext?.Span, + options: _configuration.Instrumentation); + + var arrayOfIds = new ArrayValue(); + arrayOfIds.Values.AddRange(ids.Select(id => new Value { StringValue = id })); + try + { + var query = new StructuredQuery + { + From = { new StructuredQuery.Types.CollectionSelector { CollectionId = _outboxCollection.Name } }, + Where = new StructuredQuery.Types.Filter + { + FieldFilter = new StructuredQuery.Types.FieldFilter + { + Field = new StructuredQuery.Types.FieldReference { FieldPath = nameof(MessageHeader.MessageId) }, + Op = StructuredQuery.Types.FieldFilter.Types.Operator.In, + Value = new Value { ArrayValue = arrayOfIds } + } + }, + OrderBy = + { + new StructuredQuery.Types.Order + { + Field = new StructuredQuery.Types.FieldReference { FieldPath = nameof(MessageHeader.TimeStamp) }, + Direction = StructuredQuery.Types.Direction.Ascending + } + } + }; + + var request = new RunQueryRequest + { + Parent = $"{_configuration.DatabasePath}/documents", + StructuredQuery = query + }; + + var client = _connectionProvider.GetFirestoreClient(); + return BrighterAsyncContext.Run(async () => + { + var messages = new List(); + + using var response = client.RunQuery(request); + await foreach (var doc in response.GetResponseStream()) + { + messages.Add(ToMessage(doc.Document)); + } + + return messages; + }); + } + finally + { + Tracer?.EndSpan(span); + } + } + /// public void MarkDispatched(Id id, RequestContext? requestContext, DateTimeOffset? dispatchedAt = null, Dictionary? args = null) { @@ -531,89 +602,9 @@ public IList Get(int pageSize = 100, int pageNumber = 1, Dictionary - /// Returns messages specified by the Ids - /// - /// The Ids of the messages - /// What is the context for this request; used to access the Span - /// The Timeout of the outbox. - /// A list of messages - public IList Get (IEnumerable messageIds, RequestContext? requestContext = null, int outBoxTimeout = -1) - { - var ids = messageIds.Select(id => id.Value).ToArray(); - - var dbAttributes = new Dictionary() - { - {"db.operation.parameter.message.ids", string.Join(",", ids)} - }; - - var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(DbSystem.Firestore, - _configuration.Database, - BoxDbOperation.Get, - _outboxCollection.Name, - dbAttributes: dbAttributes), - requestContext?.Span, - options: _configuration.Instrumentation); - - var arrayOfIds = new ArrayValue(); - arrayOfIds.Values.AddRange(ids.Select(id => new Value { StringValue = id})); - try - { - var query = new StructuredQuery - { - From = { new StructuredQuery.Types.CollectionSelector { CollectionId = _outboxCollection.Name } }, - Where = new StructuredQuery.Types.Filter - { - FieldFilter = new StructuredQuery.Types.FieldFilter - { - Field = new StructuredQuery.Types.FieldReference { FieldPath = nameof(MessageHeader.MessageId) }, - Op = StructuredQuery.Types.FieldFilter.Types.Operator.In, - Value = new Value { ArrayValue = arrayOfIds } - } - }, - OrderBy = - { - new StructuredQuery.Types.Order - { - Field = new StructuredQuery.Types.FieldReference { FieldPath = nameof(MessageHeader.TimeStamp) }, - Direction = StructuredQuery.Types.Direction.Ascending - } - } - }; - - var request = new RunQueryRequest - { - Parent = $"{_configuration.DatabasePath}/documents", - StructuredQuery = query - }; - - var client = _connectionProvider.GetFirestoreClient(); - return BrighterAsyncContext.Run(async () => - { - var messages = new List(); - - using var response = client.RunQuery(request); - await foreach (var doc in response.GetResponseStream()) - { - messages.Add(ToMessage(doc.Document)); - } - - return messages; - }); - } - finally - { - Tracer?.EndSpan(span); - } - } - /// - /// Get the number of messages in the Outbox that are not dispatched - /// - /// - public long GetNumberOfOutstandingMessages() + /// + public int GetOutstandingMessageCount(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null) { var span = Tracer?.CreateDbSpan( new BoxSpanInfo(DbSystem.Firestore, @@ -668,7 +659,7 @@ public long GetNumberOfOutstandingMessages() if (resp.Result.AggregateFields.TryGetValue("total_count", out var value)) { // The aggregated count is typically returned as an Int64Value in the Value proto - return value.IntegerValue; + return value.IntegerValue > int.MaxValue ? int.MaxValue : (int)value.IntegerValue; } } @@ -1158,7 +1149,7 @@ public async Task> OutstandingMessagesAsync(TimeSpan dispat /// Additional parameters required for search, if any /// /// A list of messages - public async Task> GetAsync(int pageSize = 100, int pageNumber = 1, Dictionary? args = null, CancellationToken cancellationToken = default) + public async Task> GetAsync(int pageSize = 100, int pageNumber = 1, Dictionary? args = null, CancellationToken cancellationToken = default) { var span = Tracer?.CreateDbSpan( new BoxSpanInfo(DbSystem.Firestore, _configuration.Database, BoxDbOperation.Get, _outboxCollection.Name), @@ -1228,7 +1219,7 @@ public async Task> GetAsync(int pageSize = 100, int pageNumber = /// The Timeout of the outbox. /// /// A list of messages - public async Task> GetAsync(IEnumerable messageIds, RequestContext? requestContext = null, int outBoxTimeout = -1, CancellationToken cancellationToken = default) + public async Task> GetAsync(IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null, CancellationToken cancellationToken = default) { var ids = messageIds.Select(id => id.Value).ToArray(); @@ -1294,13 +1285,9 @@ public async Task> GetAsync(IEnumerable messageIds, RequestCo Tracer?.EndSpan(span); } } - - /// - /// Get the number of messages in the Outbox that are not dispatched - /// - /// Cancel the async operation - /// - public async Task GetNumberOfOutstandingMessagesAsync(CancellationToken cancellationToken = default) + + /// + public async Task GetOutstandingMessageCountAsync(TimeSpan dispatchedSince, RequestContext? requestContext, int maxCount = 100, Dictionary? args = null, CancellationToken cancellationToken = default) { var span = Tracer?.CreateDbSpan( new BoxSpanInfo(DbSystem.Firestore, @@ -1353,7 +1340,7 @@ public async Task GetNumberOfOutstandingMessagesAsync(CancellationToken ca if (resp.Result.AggregateFields.TryGetValue("total_count", out var value)) { // The aggregated count is typically returned as an Int64Value in the Value proto - return value.IntegerValue; + return value.IntegerValue > int.MaxValue ? int.MaxValue : (int)value.IntegerValue; } } diff --git a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages.cs b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages.cs index f285f81c94..2a5a336b6c 100644 --- a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages.cs +++ b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using Paramore.Brighter.Outbox.Firestore; namespace Paramore.Brighter.Gcp.Tests.Firestore.Outbox; @@ -58,7 +59,7 @@ public void When_Retrieving_Messages_By_Id() //Assert Assert.NotNull(messages); - Assert.Equal(2, messages.Count); + Assert.Equal(2, messages.Count()); Assert.Contains(messages, message => message.Id == _messageEarliest.Id); Assert.Contains(messages, message => message.Id == _messageUnDispatched.Id); Assert.Contains(messages, message => message.Id == _messageUnDispatched.Id); diff --git a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages_async.cs b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages_async.cs index 4ec30bca29..284b39e5da 100644 --- a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages_async.cs +++ b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_messages_async.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using Paramore.Brighter.Outbox.Firestore; @@ -39,7 +40,7 @@ public async Task When_Retrieving_Messages_Async() var messages = await _outbox.GetAsync(pageSize: 1_000); //Assert - Assert.True(messages.Count >= 3); + Assert.True(messages.Count() >= 3); Assert.Contains(messages, message => message.Id == _messageEarliest.Id); Assert.Contains(messages, message => message.Id == _messageDispatched.Id); Assert.Contains(messages, message => message.Id == _messageUnDispatched.Id); @@ -60,7 +61,7 @@ public async Task When_Retrieving_Messages_By_Id_Async() //Assert //Assert Assert.NotNull(messages); - Assert.Equal(2, messages.Count); + Assert.Equal(2, messages.Count()); Assert.Contains(messages, message => message.Id == _messageEarliest.Id); Assert.Contains(messages, message => message.Id == _messageUnDispatched.Id); Assert.Contains(messages, message => message.Id == _messageUnDispatched.Id); diff --git a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages.cs b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages.cs index 05ecb56c95..7b8158afe9 100644 --- a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages.cs +++ b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages.cs @@ -38,14 +38,14 @@ public void When_Retrieving_Not_Dispatched_Messages() _outbox.Add([_messageEarliest, _messageDispatched, _messageUnDispatched], context); _outbox.MarkDispatched(_messageDispatched.Id, context); - var total = _outbox.GetNumberOfOutstandingMessages(); + var total = _outbox.GetOutstandingMessageCount(TimeSpan.Zero, null); var allUnDispatched = _outbox.OutstandingMessages(TimeSpan.Zero, context); var messagesOverAnHour = _outbox.OutstandingMessages(TimeSpan.FromHours(1), context); var messagesOver4Hours = _outbox.OutstandingMessages(TimeSpan.FromHours(4), context); //Assert - Assert.True(total >= 2); + Assert.True(total >= 2); allUnDispatched = allUnDispatched.ToList(); Assert.True(allUnDispatched.Count() > 2); diff --git a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages_async.cs b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages_async.cs index 6fae2d1102..cc59ecabcf 100644 --- a/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages_async.cs +++ b/tests/Paramore.Brighter.Gcp.Tests/Firestore/Outbox/When_retrieving_outstanding_messages_async.cs @@ -38,8 +38,8 @@ public async Task When_Retrieving_Not_Dispatched_Messages_Async() var context = new RequestContext(); await _outbox.AddAsync([_messageEarliest, _messageDispatched, _messageUnDispatched], context); await _outbox.MarkDispatchedAsync(_messageDispatched.Id, context); - - var total = await _outbox.GetNumberOfOutstandingMessagesAsync(); + + var total = await _outbox.GetOutstandingMessageCountAsync(TimeSpan.Zero, context); var allUnDispatched = await _outbox.OutstandingMessagesAsync(TimeSpan.Zero, context, pageSize: 1_000); var messagesOverAnHour = await _outbox.OutstandingMessagesAsync(TimeSpan.FromHours(1), context, pageSize: 1_000);