From 83caa2f25d590d0cf0a8e6fb68c2dc44ceabd21c Mon Sep 17 00:00:00 2001 From: Rafael Andrade Date: Fri, 11 Jul 2025 12:31:49 +0100 Subject: [PATCH 1/3] feat: Improve inbox and outbox --- .../MsSqlInbox.cs | 245 +---- .../Paramore.Brighter.Inbox.MsSql.csproj | 1 + .../MySqlInbox.cs | 247 +---- .../Paramore.Brighter.Inbox.MySql.csproj | 5 +- .../Paramore.Brighter.Inbox.Postgres.csproj | 3 +- .../PostgreSqlInbox.cs | 226 +---- .../PostgreSqlQueries.cs | 6 +- .../Paramore.Brighter.Inbox.Sqlite.csproj | 1 + .../SqliteInbox.cs | 244 +---- .../SqliteQueries.cs | 6 +- .../MsSqlOutbox.cs | 714 +------------- .../MySqlOutbox.cs | 507 +--------- .../Paramore.Brighter.Outbox.MySql.csproj | 1 + .../PostgreSqlOutbox.cs | 701 +------------ .../SqliteOutbox.cs | 685 +------------ .../RelationDatabaseOutbox.cs | 928 +++++++++++++++--- .../RelationalDatabaseInbox.cs | 339 +++++-- 17 files changed, 1344 insertions(+), 3515 deletions(-) diff --git a/src/Paramore.Brighter.Inbox.MsSql/MsSqlInbox.cs b/src/Paramore.Brighter.Inbox.MsSql/MsSqlInbox.cs index c6f6706f01..c12239ef33 100644 --- a/src/Paramore.Brighter.Inbox.MsSql/MsSqlInbox.cs +++ b/src/Paramore.Brighter.Inbox.MsSql/MsSqlInbox.cs @@ -25,229 +25,50 @@ THE SOFTWARE. */ using System; using System.Data; -using System.Data.Common; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Data.SqlClient; -using Paramore.Brighter.Inbox.Exceptions; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.MsSql; using Paramore.Brighter.Observability; -namespace Paramore.Brighter.Inbox.MsSql +namespace Paramore.Brighter.Inbox.MsSql; + +/// +/// Class MsSqlInbox. +/// +public class MsSqlInbox : RelationalDatabaseInbox { + private const int MsSqlDuplicateKeyError_UniqueIndexViolation = 2601; + private const int MsSqlDuplicateKeyError_UniqueConstraintViolation = 2627; + /// - /// Class MsSqlInbox. + /// Initializes a new instance of the class. /// - public class MsSqlInbox : RelationalDatabaseInbox + /// The configuration. + /// The Connection Provider. + public MsSqlInbox(IAmARelationalDatabaseConfiguration configuration, IAmARelationalDbConnectionProvider connectionProvider) + : base(DbSystem.MsSql, configuration, connectionProvider, + new MsSqlQueries(), ApplicationLogging.CreateLogger()) { - private const int MsSqlDuplicateKeyError_UniqueIndexViolation = 2601; - private const int MsSqlDuplicateKeyError_UniqueConstraintViolation = 2627; - private readonly IAmARelationalDbConnectionProvider _connectionProvider; - - /// - /// Initializes a new instance of the class. - /// - /// The configuration. - /// The Connection Provider. - public MsSqlInbox(IAmARelationalDatabaseConfiguration configuration, IAmARelationalDbConnectionProvider connectionProvider) - : base(DbSystem.MsSql, configuration.DatabaseName, configuration.InBoxTableName, - new MsSqlQueries(), ApplicationLogging.CreateLogger()) - { - ContinueOnCapturedContext = false; - _connectionProvider = connectionProvider; - } - - /// - /// Initializes a new instance of the class. - /// - /// The configuration. - public MsSqlInbox(IAmARelationalDatabaseConfiguration configuration) : this(configuration, - new MsSqlConnectionProvider(configuration)) - { - } - - protected override DbCommand CreateCommand( - DbConnection connection, string sqlText, int outBoxTimeout, params IDbDataParameter[] parameters) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override void WriteToStore(Func commandFunc, Action loggingAction) - { - using var connection = GetOpenConnection(_connectionProvider); - using var command = commandFunc.Invoke(connection); - try - { - command.ExecuteNonQuery(); - } - catch (SqlException ex) - { - if (ex.Number == MsSqlDuplicateKeyError_UniqueIndexViolation || ex.Number == MsSqlDuplicateKeyError_UniqueConstraintViolation) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override async Task WriteToStoreAsync(Func commandFunc, - Action loggingAction, CancellationToken cancellationToken) - { - using var connection = await GetOpenConnectionAsync(_connectionProvider, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - try - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - } - catch (SqlException ex) - { - if (ex.Number == MsSqlDuplicateKeyError_UniqueIndexViolation || ex.Number == MsSqlDuplicateKeyError_UniqueConstraintViolation) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override T ReadFromStore(Func commandFunc, Func resultFunc, string commandId) - { - using var connection = _connectionProvider.GetConnection(); - using var command = commandFunc.Invoke(connection); - - var result = command.ExecuteReader(); - return resultFunc.Invoke(result, commandId); - } - - protected override async Task ReadFromStoreAsync(Func commandFunc, - Func> resultFunc, - string commandId, - CancellationToken cancellationToken) - { - using var connection = await _connectionProvider.GetConnectionAsync(cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - - var result = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - return await resultFunc.Invoke(result, commandId, cancellationToken); - } - - protected override IDbDataParameter[] CreateAddParameters(T command, string contextKey) - { - var commandJson = JsonSerializer.Serialize(command, JsonSerialisationOptions.Options); - var parameters = new[] - { - CreateSqlParameter("CommandID", command.Id.Value), - CreateSqlParameter("CommandType", typeof (T).Name), - CreateSqlParameter("CommandBody", commandJson), - CreateSqlParameter("Timestamp", DateTime.UtcNow), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateGetParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateSqlParameter("CommandID", commandId), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateExistsParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateSqlParameter("CommandID", commandId), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - private SqlParameter CreateSqlParameter(string parameterName, object value) - { - return new SqlParameter(parameterName, value ?? DBNull.Value); - } - - protected override T MapFunction(DbDataReader dr, string commandId) - { - try - { - if (dr.Read()) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { - dr.Close(); - } - - throw new RequestNotFoundException(commandId); - } - - protected override async Task MapFunctionAsync(DbDataReader dr, string commandId, - CancellationToken cancellationToken) - { - try - { - if (await dr.ReadAsync().ConfigureAwait(ContinueOnCapturedContext)) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { -#if NET462 - dr.Close(); -#else - await dr.CloseAsync().ConfigureAwait(ContinueOnCapturedContext); -#endif - } + } - throw new RequestNotFoundException(commandId); - } + /// + /// Initializes a new instance of the class. + /// + /// The configuration. + public MsSqlInbox(IAmARelationalDatabaseConfiguration configuration) : this(configuration, + new MsSqlConnectionProvider(configuration)) + { + } - protected override bool MapBoolFunction(DbDataReader dr, string commandId) - { - try - { - return dr.HasRows; - } - finally - { - dr.Close(); - } - } + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) + { + return ex is SqlException { Number: MsSqlDuplicateKeyError_UniqueIndexViolation or MsSqlDuplicateKeyError_UniqueConstraintViolation }; + } - protected override Task MapBoolFunctionAsync(DbDataReader dr, string commandId, - CancellationToken cancellationToken) - { - try - { - return Task.FromResult(dr.HasRows); - } - finally - { - dr.Close(); - } - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) + { + return new SqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; } } diff --git a/src/Paramore.Brighter.Inbox.MsSql/Paramore.Brighter.Inbox.MsSql.csproj b/src/Paramore.Brighter.Inbox.MsSql/Paramore.Brighter.Inbox.MsSql.csproj index e606fa5b63..cb6cd6c9f1 100644 --- a/src/Paramore.Brighter.Inbox.MsSql/Paramore.Brighter.Inbox.MsSql.csproj +++ b/src/Paramore.Brighter.Inbox.MsSql/Paramore.Brighter.Inbox.MsSql.csproj @@ -4,6 +4,7 @@ Francesco Pighi $(BrighterFrameworkAndCoreTargetFrameworks) RabbitMQ;AMQP;Command;Event;Service Activator;Decoupled;Invocation;Messaging;Remote;Command Dispatcher;Command Processor;Request;Service;Task Queue;Work Queue;Retry;Circuit Breaker;Availability + enable diff --git a/src/Paramore.Brighter.Inbox.MySql/MySqlInbox.cs b/src/Paramore.Brighter.Inbox.MySql/MySqlInbox.cs index 9af62f26bd..6fa3e51a00 100644 --- a/src/Paramore.Brighter.Inbox.MySql/MySqlInbox.cs +++ b/src/Paramore.Brighter.Inbox.MySql/MySqlInbox.cs @@ -25,232 +25,49 @@ THE SOFTWARE. */ using System; using System.Data; -using System.Data.Common; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using MySqlConnector; -using Paramore.Brighter.Inbox.Exceptions; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.MySql; using Paramore.Brighter.Observability; -namespace Paramore.Brighter.Inbox.MySql +namespace Paramore.Brighter.Inbox.MySql; + +/// +/// Class MySqlInbox. +/// +public class MySqlInbox : RelationalDatabaseInbox { + private const int MySqlDuplicateKeyError = 1062; + /// - /// Class MySqlInbox. + /// Initializes a new instance of the class. /// - public class MySqlInbox : RelationalDatabaseInbox + /// The configuration. + /// The Connection Provider. + public MySqlInbox(IAmARelationalDatabaseConfiguration configuration, IAmARelationalDbConnectionProvider connectionProvider) + : base(DbSystem.MySql, configuration, connectionProvider, + new MySqlQueries(), ApplicationLogging.CreateLogger()) { - private const int MySqlDuplicateKeyError = 1062; - private readonly IAmARelationalDbConnectionProvider _connectionProvider; - - /// - /// Initializes a new instance of the class. - /// - /// The configuration. - /// The Connection Provider. - public MySqlInbox(IAmARelationalDatabaseConfiguration configuration, IAmARelationalDbConnectionProvider connectionProvider) - : base(DbSystem.MySql, configuration.DatabaseName, configuration.InBoxTableName, - new MySqlQueries(), ApplicationLogging.CreateLogger()) - { - ContinueOnCapturedContext = false; - _connectionProvider = connectionProvider; - } - - /// - /// Initializes a new instance of the class. - /// - /// The configuration. - public MySqlInbox(IAmARelationalDatabaseConfiguration configuration) : this(configuration, - new MySqlConnectionProvider(configuration)) - { - } - - protected override DbCommand CreateCommand( - DbConnection connection, string sqlText, int outBoxTimeout, params IDbDataParameter[] parameters) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override void WriteToStore(Func commandFunc, Action loggingAction) - { - using var connection = GetOpenConnection(_connectionProvider); - using var command = commandFunc.Invoke(connection); - try - { - command.ExecuteNonQuery(); - } - catch (MySqlException ex) - { - if (ex.Number == MySqlDuplicateKeyError) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override async Task WriteToStoreAsync(Func commandFunc, - Action loggingAction, CancellationToken cancellationToken) - { - using var connection = await GetOpenConnectionAsync(_connectionProvider, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - try - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - } - catch (MySqlException ex) - { - if (ex.Number == MySqlDuplicateKeyError) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override T ReadFromStore(Func commandFunc, Func resultFunc, string commandId) - { - using var connection = _connectionProvider.GetConnection(); - using var command = commandFunc.Invoke(connection); - - var result = command.ExecuteReader(); - return resultFunc.Invoke(result, commandId); - } - - protected override async Task ReadFromStoreAsync(Func commandFunc, - Func> resultFunc, - string commandId, - CancellationToken cancellationToken) - { - using var connection = await _connectionProvider.GetConnectionAsync(cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - - var result = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - return await resultFunc.Invoke(result, commandId, cancellationToken); - } - - protected override IDbDataParameter[] CreateAddParameters(T command, string contextKey) - { - var commandJson = JsonSerializer.Serialize(command, JsonSerialisationOptions.Options); - var parameters = new[] - { - CreateSqlParameter("CommandID", command.Id.Value), - CreateSqlParameter("CommandType", typeof (T).Name), - CreateSqlParameter("CommandBody", commandJson), - CreateSqlParameter("Timestamp", DateTime.UtcNow), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateGetParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateSqlParameter("CommandID", commandId), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateExistsParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateSqlParameter("CommandID", commandId), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - private DbParameter CreateSqlParameter(string parameterName, object value) - { - return new MySqlParameter - { - ParameterName = parameterName, - Value = value - }; - } - - protected override T MapFunction(DbDataReader dr, string commandId) - { - try - { - if (dr.Read()) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { - dr.Close(); - } - - throw new RequestNotFoundException(commandId); - } - - protected override async Task MapFunctionAsync(DbDataReader dr, string commandId, - CancellationToken cancellationToken) - { - try - { - if (await dr.ReadAsync().ConfigureAwait(ContinueOnCapturedContext)) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { -#if NETSTANDARD2_0 - dr.Close(); -#else - await dr.CloseAsync().ConfigureAwait(ContinueOnCapturedContext); -#endif - } + } - throw new RequestNotFoundException(commandId); - } + /// + /// Initializes a new instance of the class. + /// + /// The configuration. + public MySqlInbox(IAmARelationalDatabaseConfiguration configuration) : this(configuration, + new MySqlConnectionProvider(configuration)) + { + } - protected override bool MapBoolFunction(DbDataReader dr, string commandId) - { - try - { - return dr.HasRows; - } - finally - { - dr.Close(); - } - } + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) + { + return ex is MySqlException { Number: MySqlDuplicateKeyError }; + } - protected override Task MapBoolFunctionAsync(DbDataReader dr, string commandId, - CancellationToken cancellationToken) - { - try - { - return Task.FromResult(dr.HasRows); - } - finally - { - dr.Close(); - } - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) + { + return new MySqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; } } diff --git a/src/Paramore.Brighter.Inbox.MySql/Paramore.Brighter.Inbox.MySql.csproj b/src/Paramore.Brighter.Inbox.MySql/Paramore.Brighter.Inbox.MySql.csproj index 8c72838302..6d8119048b 100644 --- a/src/Paramore.Brighter.Inbox.MySql/Paramore.Brighter.Inbox.MySql.csproj +++ b/src/Paramore.Brighter.Inbox.MySql/Paramore.Brighter.Inbox.MySql.csproj @@ -4,14 +4,15 @@ Derek Comartin $(BrighterTargetFrameworks) MySql;Command;Event;Service Activator;Decoupled;Invocation;Messaging;Remote;Command Dispatcher;Command Processor;Request;Service;Task Queue;Work Queue;Retry;Circuit Breaker;Availability + enable + + - - \ No newline at end of file diff --git a/src/Paramore.Brighter.Inbox.Postgres/Paramore.Brighter.Inbox.Postgres.csproj b/src/Paramore.Brighter.Inbox.Postgres/Paramore.Brighter.Inbox.Postgres.csproj index cfcc260dda..e0bb124651 100644 --- a/src/Paramore.Brighter.Inbox.Postgres/Paramore.Brighter.Inbox.Postgres.csproj +++ b/src/Paramore.Brighter.Inbox.Postgres/Paramore.Brighter.Inbox.Postgres.csproj @@ -3,7 +3,8 @@ This is an implementation of the inbox used for decoupled invocation of commands by Paramore.Brighter, using PostgreSql $(BrighterCoreTargetFrameworks) - MySql;Command;Event;Service Activator;Decoupled;Invocation;Messaging;Remote;Command Dispatcher;Command Processor;Request;Service;Task Queue;Work Queue;Retry;Circuit Breaker;Availability + Postgres;Command;Event;Service Activator;Decoupled;Invocation;Messaging;Remote;Command Dispatcher;Command Processor;Request;Service;Task Queue;Work Queue;Retry;Circuit Breaker;Availability + enable diff --git a/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlInbox.cs b/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlInbox.cs index 06cbf7337e..711daae670 100644 --- a/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlInbox.cs +++ b/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlInbox.cs @@ -25,222 +25,36 @@ THE SOFTWARE. */ using System; using System.Data; -using System.Data.Common; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Npgsql; -using NpgsqlTypes; -using Paramore.Brighter.Inbox.Exceptions; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.Observability; using Paramore.Brighter.PostgreSql; -namespace Paramore.Brighter.Inbox.Postgres +namespace Paramore.Brighter.Inbox.Postgres; + +public class PostgreSqlInbox : RelationalDatabaseInbox { - public class PostgreSqlInbox : RelationalDatabaseInbox + public PostgreSqlInbox(IAmARelationalDatabaseConfiguration configuration, IAmARelationalDbConnectionProvider connectionProvider) + : base(DbSystem.Postgresql, configuration, connectionProvider, + new PostgreSqlQueries(), ApplicationLogging.CreateLogger()) { - private readonly IAmARelationalDatabaseConfiguration _configuration; - private readonly IAmARelationalDbConnectionProvider _connectionProvider; - - public PostgreSqlInbox(IAmARelationalDbConnectionProvider connectionProvider, IAmARelationalDatabaseConfiguration configuration) - : base(DbSystem.Postgresql, configuration.DatabaseName, configuration.InBoxTableName, - new PostgreSqlQueries(), ApplicationLogging.CreateLogger()) - { - _connectionProvider = connectionProvider; - _configuration = configuration; - ContinueOnCapturedContext = false; - } - - public PostgreSqlInbox(IAmARelationalDatabaseConfiguration configuration) - : this(new PostgreSqlConnectionProvider(configuration), configuration) - { - } - - protected override void WriteToStore(Func commandFunc, Action loggingAction) - { - using var connection = GetOpenConnection(_connectionProvider); - using var command = commandFunc.Invoke(connection); - try - { - command.ExecuteNonQuery(); - } - catch (PostgresException ex) - { - if (ex.SqlState == PostgresErrorCodes.UniqueViolation) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override async Task WriteToStoreAsync(Func commandFunc, Action loggingAction, CancellationToken cancellationToken) - { - using var connection = await GetOpenConnectionAsync(_connectionProvider, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - try - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - } - catch (PostgresException ex) - { - if (ex.SqlState == PostgresErrorCodes.UniqueViolation) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override T ReadFromStore(Func commandFunc, Func resultFunc, string commandId) - { - using var connection = _connectionProvider.GetConnection(); - using var command = commandFunc.Invoke(connection); - - var result = command.ExecuteReader(); - return resultFunc.Invoke(result, commandId); - } - - protected override async Task ReadFromStoreAsync(Func commandFunc, - Func> resultFunc, - string commandId, - CancellationToken cancellationToken) - { - using var connection = await _connectionProvider.GetConnectionAsync(cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - - var result = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - return await resultFunc.Invoke(result, commandId, cancellationToken); - } - - protected override DbCommand CreateCommand( - DbConnection connection, string sqlText, int outBoxTimeout, params IDbDataParameter[] parameters) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override IDbDataParameter[] CreateAddParameters(T command, string contextKey) - { - var commandJson = JsonSerializer.Serialize(command, JsonSerialisationOptions.Options); - var parameters = new[] - { - CreateNpgsqlParameter("CommandID", command.Id.Value), - CreateNpgsqlParameter("CommandType", typeof (T).Name), - CreateNpgsqlParameter("CommandBody", commandJson), - CreateNpgsqlParameter("Timestamp", NpgsqlDbType.TimestampTz, DateTimeOffset.UtcNow), - CreateNpgsqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateExistsParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateNpgsqlParameter("CommandId", commandId), - CreateNpgsqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateGetParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateNpgsqlParameter("CommandId", commandId), - CreateNpgsqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - private NpgsqlParameter CreateNpgsqlParameter(string parameterName, object value) - { - if (value != null) - return new NpgsqlParameter(parameterName, value); - else - return new NpgsqlParameter(parameterName, DBNull.Value); - } - - private NpgsqlParameter CreateNpgsqlParameter(string parameterName, NpgsqlDbType dbType, object value) - { - if (value != null) - return new NpgsqlParameter(parameterName, dbType) { Value = value }; - else - return new NpgsqlParameter(parameterName, dbType) { Value = DBNull.Value }; - } - - protected override T MapFunction(DbDataReader dr, string commandId) - { - try - { - if (dr.Read()) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { - dr.Close(); - } - - throw new RequestNotFoundException(commandId); - } + } - protected override async Task MapFunctionAsync(DbDataReader dr, string commandId, CancellationToken cancellationToken) - { - try - { - if (await dr.ReadAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext)) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { - await dr.CloseAsync().ConfigureAwait(ContinueOnCapturedContext); - } + public PostgreSqlInbox(IAmARelationalDatabaseConfiguration configuration) + : this(configuration, new PostgreSqlConnectionProvider(configuration)) + { + } - throw new RequestNotFoundException(commandId); - } - protected override bool MapBoolFunction(DbDataReader dr, string commandId) - { - try - { - return dr.HasRows; - } - finally - { - dr.Close(); - } - } + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) + { + return ex is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation }; + } - protected override Task MapBoolFunctionAsync(DbDataReader dr, string commandId, CancellationToken cancellationToken) - { - try - { - return Task.FromResult(dr.HasRows); - } - finally - { - dr.Close(); - } - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) + { + return new NpgsqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; } } diff --git a/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlQueries.cs b/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlQueries.cs index 8c0a43fe2d..81dfe846a2 100644 --- a/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlQueries.cs +++ b/src/Paramore.Brighter.Inbox.Postgres/PostgreSqlQueries.cs @@ -27,10 +27,10 @@ namespace Paramore.Brighter.Inbox.Postgres { public class PostgreSqlQueries : IRelationalDatabaseInboxQueries { - public string AddCommand { get; } = "INSERT INTO {0} (CommandId, CommandType, CommandBody, Timestamp, ContextKey) VALUES (@CommandId, @CommandType, @CommandBody, @Timestamp, @ContextKey)"; + public string AddCommand { get; } = "INSERT INTO {0} (CommandId, CommandType, CommandBody, Timestamp, ContextKey) VALUES (@CommandID, @CommandType, @CommandBody, @Timestamp, @ContextKey)"; - public string ExistsCommand { get; } = "SELECT DISTINCT CommandId FROM {0} WHERE CommandId = @CommandId AND ContextKey = @ContextKey FETCH FIRST 1 ROWS ONLY"; + public string ExistsCommand { get; } = "SELECT DISTINCT CommandId FROM {0} WHERE CommandId = @CommandID AND ContextKey = @ContextKey FETCH FIRST 1 ROWS ONLY"; - public string GetCommand { get; } = "SELECT * FROM {0} WHERE CommandId = @CommandId AND ContextKey = @ContextKey"; + public string GetCommand { get; } = "SELECT * FROM {0} WHERE CommandId = @CommandID AND ContextKey = @ContextKey"; } } diff --git a/src/Paramore.Brighter.Inbox.Sqlite/Paramore.Brighter.Inbox.Sqlite.csproj b/src/Paramore.Brighter.Inbox.Sqlite/Paramore.Brighter.Inbox.Sqlite.csproj index 4b1bdd03be..ccb07730a4 100644 --- a/src/Paramore.Brighter.Inbox.Sqlite/Paramore.Brighter.Inbox.Sqlite.csproj +++ b/src/Paramore.Brighter.Inbox.Sqlite/Paramore.Brighter.Inbox.Sqlite.csproj @@ -4,6 +4,7 @@ Ian Cooper $(BrighterTargetFrameworks) RabbitMQ;AMQP;Command;Event;Service Activator;Decoupled;Invocation;Messaging;Remote;Command Dispatcher;Command Processor;Request;Service;Task Queue;Work Queue;Retry;Circuit Breaker;Availability + enable diff --git a/src/Paramore.Brighter.Inbox.Sqlite/SqliteInbox.cs b/src/Paramore.Brighter.Inbox.Sqlite/SqliteInbox.cs index a936c37c19..3dfa940fdf 100644 --- a/src/Paramore.Brighter.Inbox.Sqlite/SqliteInbox.cs +++ b/src/Paramore.Brighter.Inbox.Sqlite/SqliteInbox.cs @@ -25,228 +25,50 @@ THE SOFTWARE. */ using System; using System.Data; -using System.Data.Common; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Data.Sqlite; -using Paramore.Brighter.Inbox.Exceptions; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.Observability; using Paramore.Brighter.Sqlite; -namespace Paramore.Brighter.Inbox.Sqlite +namespace Paramore.Brighter.Inbox.Sqlite; + +/// +/// Class SqliteInbox. +/// +public class SqliteInbox : RelationalDatabaseInbox { + private const int SqliteDuplicateKeyError = 1555; + private const int SqliteUniqueKeyError = 19; + /// - /// Class SqliteInbox. + /// Initializes a new instance of the class. /// - public class SqliteInbox : RelationalDatabaseInbox + /// The connection provider for the database. + /// The configuration for the database. + public SqliteInbox(IAmARelationalDatabaseConfiguration configuration, IAmARelationalDbConnectionProvider connectionProvider) + : base(DbSystem.Sqlite, configuration, connectionProvider, + new SqliteQueries(), ApplicationLogging.CreateLogger()) { - private readonly IAmARelationalDbConnectionProvider _connectionProvider; - private const int SqliteDuplicateKeyError = 1555; - private const int SqliteUniqueKeyError = 19; - - /// - /// Initializes a new instance of the class. - /// - /// The connection provider for the database. - /// The configuration for the database. - public SqliteInbox(IAmARelationalDbConnectionProvider connectionProvider, IAmARelationalDatabaseConfiguration configuration) - : base(DbSystem.Sqlite, configuration.DatabaseName, configuration.InBoxTableName, - new SqliteQueries(), ApplicationLogging.CreateLogger()) - { - _connectionProvider = connectionProvider; - ContinueOnCapturedContext = false; - } - - /// - /// Initializes a new instance of the class. - /// - /// The configuration for the database. - public SqliteInbox(IAmARelationalDatabaseConfiguration configuration) - : this(new SqliteConnectionProvider(configuration), configuration) - { - } - - protected override void WriteToStore(Func commandFunc, Action loggingAction) - { - using var connection = GetOpenConnection(_connectionProvider); - using var command = commandFunc.Invoke(connection); - try - { - command.ExecuteNonQuery(); - } - catch (SqliteException ex) - { - if (ex.SqliteErrorCode == SqliteDuplicateKeyError || - ex.SqliteErrorCode == SqliteUniqueKeyError) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override async Task WriteToStoreAsync(Func commandFunc, Action loggingAction, CancellationToken cancellationToken) - { - using var connection = await GetOpenConnectionAsync(_connectionProvider, cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - try - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - } - catch (SqliteException ex) - { - if (ex.SqliteErrorCode == SqliteDuplicateKeyError || - ex.SqliteErrorCode == SqliteUniqueKeyError) - { - loggingAction.Invoke(); - return; - } - - throw; - } - } - - protected override T ReadFromStore(Func commandFunc, Func resultFunc, string commandId) - { - using var connection = _connectionProvider.GetConnection(); - using var command = commandFunc.Invoke(connection); - - var result = command.ExecuteReader(); - return resultFunc.Invoke(result, commandId); - } - - protected override async Task ReadFromStoreAsync(Func commandFunc, - Func> resultFunc, - string commandId, - CancellationToken cancellationToken) - { - using var connection = await _connectionProvider.GetConnectionAsync(cancellationToken) - .ConfigureAwait(ContinueOnCapturedContext); - using var command = commandFunc.Invoke(connection); - - var result = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext); - return await resultFunc.Invoke(result, commandId, cancellationToken); - } - - protected override DbCommand CreateCommand(DbConnection connection, string sqlText, int outBoxTimeout, params IDbDataParameter[] parameters) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override IDbDataParameter[] CreateAddParameters(T command, string contextKey) - { - var commandJson = JsonSerializer.Serialize(command, JsonSerialisationOptions.Options); - var parameters = new[] - { - CreateSqlParameter("CommandId", command.Id.Value), //was CommandId - CreateSqlParameter("CommandType", typeof (T).Name), - CreateSqlParameter("CommandBody", commandJson), - CreateSqlParameter("Timestamp", DateTime.UtcNow), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateExistsParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateSqlParameter("CommandId", commandId), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - protected override IDbDataParameter[] CreateGetParameters(string commandId, string contextKey) - { - var parameters = new[] - { - CreateSqlParameter("CommandId", commandId), - CreateSqlParameter("ContextKey", contextKey) - }; - return parameters; - } - - private DbParameter CreateSqlParameter(string parameterName, object value) - { - return new SqliteParameter(parameterName, value); - } - - protected override T MapFunction(DbDataReader dr, string commandId) - { - try - { - if (dr.Read()) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { - dr.Close(); - } - - throw new RequestNotFoundException(commandId); - } - - protected override async Task MapFunctionAsync(DbDataReader dr, string commandId, - CancellationToken cancellationToken) - { - try - { - if (await dr.ReadAsync().ConfigureAwait(ContinueOnCapturedContext)) - { - var body = dr.GetString(dr.GetOrdinal("CommandBody")); - return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options); - } - } - finally - { -#if NETSTANDARD2_0 - dr.Close(); -#else - await dr.CloseAsync().ConfigureAwait(ContinueOnCapturedContext); -#endif - } + } - throw new RequestNotFoundException(commandId); - } + /// + /// Initializes a new instance of the class. + /// + /// The configuration for the database. + public SqliteInbox(IAmARelationalDatabaseConfiguration configuration) + : this(configuration, new SqliteConnectionProvider(configuration)) + { + } - protected override bool MapBoolFunction(DbDataReader dr, string commandId) - { - try - { - return dr.HasRows; - } - finally - { - dr.Close(); - } - } + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) + { + return ex is SqliteException { SqliteErrorCode: SqliteDuplicateKeyError or SqliteUniqueKeyError }; + } - protected override Task MapBoolFunctionAsync(DbDataReader dr, string commandId, CancellationToken cancellationToken) - { - try - { - return Task.FromResult(dr.HasRows); - } - finally - { - dr.Close(); - } - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) + { + return new SqliteParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; } } diff --git a/src/Paramore.Brighter.Inbox.Sqlite/SqliteQueries.cs b/src/Paramore.Brighter.Inbox.Sqlite/SqliteQueries.cs index 93f6dd7a01..2a528b2371 100644 --- a/src/Paramore.Brighter.Inbox.Sqlite/SqliteQueries.cs +++ b/src/Paramore.Brighter.Inbox.Sqlite/SqliteQueries.cs @@ -27,10 +27,10 @@ namespace Paramore.Brighter.Inbox.Sqlite { public class SqliteQueries : IRelationalDatabaseInboxQueries { - public string AddCommand { get; } = "INSERT INTO {0} (CommandId, CommandType, CommandBody, Timestamp, ContextKey) values (@CommandId, @CommandType, @CommandBody, @Timestamp, @ContextKey)"; + public string AddCommand { get; } = "INSERT INTO {0} (CommandId, CommandType, CommandBody, Timestamp, ContextKey) values (@CommandID, @CommandType, @CommandBody, @Timestamp, @ContextKey)"; - public string ExistsCommand { get; } = "SELECT CommandId FROM {0} WHERE CommandId = @CommandId and ContextKey = @ContextKey LIMIT 1"; + public string ExistsCommand { get; } = "SELECT CommandId FROM {0} WHERE CommandId = @CommandID and ContextKey = @ContextKey LIMIT 1"; - public string GetCommand { get; } = "SELECT * FROM {0} WHERE CommandId = @CommandId AND ContextKey = @ContextKey"; + public string GetCommand { get; } = "SELECT * FROM {0} WHERE CommandId = @CommandID AND ContextKey = @ContextKey"; } } diff --git a/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs b/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs index cc451a5980..59e719aa9c 100644 --- a/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs +++ b/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs @@ -24,688 +24,72 @@ THE SOFTWARE. */ #endregion using System; -using System.Collections.Generic; using System.Data; using System.Data.Common; -using System.IO; -using System.Net.Mime; using Microsoft.Data.SqlClient; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.MsSql; using Paramore.Brighter.Observability; -namespace Paramore.Brighter.Outbox.MsSql +namespace Paramore.Brighter.Outbox.MsSql; + +/// +/// Implements an Outbox using MSSQL as a backing store +/// +public class MsSqlOutbox : RelationDatabaseOutbox { + private const int MsSqlDuplicateKeyError_UniqueIndexViolation = 2601; + private const int MsSqlDuplicateKeyError_UniqueConstraintViolation = 2627; + /// - /// Implements an Outbox using MSSQL as a backing store + /// Initializes a new instance of the class. /// - public class MsSqlOutbox : RelationDatabaseOutbox + /// The configuration. + /// The connection factory. + public MsSqlOutbox(IAmARelationalDatabaseConfiguration configuration, + IAmARelationalDbConnectionProvider connectionProvider) + : base(DbSystem.MsSql, configuration, connectionProvider, + new MsSqlQueries(), ApplicationLogging.CreateLogger()) { - private const int MsSqlDuplicateKeyError_UniqueIndexViolation = 2601; - private const int MsSqlDuplicateKeyError_UniqueConstraintViolation = 2627; - private readonly IAmARelationalDatabaseConfiguration _configuration; - private readonly IAmARelationalDbConnectionProvider _connectionProvider; - - /// - /// Initializes a new instance of the class. - /// - /// The configuration. - /// The connection factory. - public MsSqlOutbox(IAmARelationalDatabaseConfiguration configuration, - IAmARelationalDbConnectionProvider connectionProvider) : base(DbSystem.MySql, configuration.DatabaseName, - configuration.OutBoxTableName, new MsSqlQueries(), ApplicationLogging.CreateLogger()) - { - _configuration = configuration; - ContinueOnCapturedContext = false; - _connectionProvider = connectionProvider; - } - - /// - /// Initializes a new instance of the class. - /// - /// The configuration. - public MsSqlOutbox(IAmARelationalDatabaseConfiguration configuration) : this(configuration, - new MsSqlConnectionProvider(configuration)) - { - } - - protected override void WriteToStore( - IAmABoxTransactionProvider? transactionProvider, - Func commandFunc, - Action? loggingAction) - { - var connection = GetOpenConnection(_connectionProvider, transactionProvider); - using var command = commandFunc.Invoke(connection); - try - { - if (transactionProvider is { HasOpenTransaction: true }) - command.Transaction = transactionProvider.GetTransaction(); - command.ExecuteNonQuery(); - } - catch (SqlException sqlException) - { - if (sqlException.Number != MsSqlDuplicateKeyError_UniqueIndexViolation && - sqlException.Number != MsSqlDuplicateKeyError_UniqueConstraintViolation) throw; - loggingAction?.Invoke(); - } - finally - { - FinishWrite(connection, transactionProvider); - } - } - - protected override async Task WriteToStoreAsync( - IAmABoxTransactionProvider? transactionProvider, - Func commandFunc, - Action? loggingAction, - CancellationToken cancellationToken) - { - var connection = await GetOpenConnectionAsync(_connectionProvider, transactionProvider, cancellationToken); - using var command = commandFunc.Invoke(connection); - try - { - if (transactionProvider is { HasOpenTransaction: true }) - command.Transaction = await transactionProvider.GetTransactionAsync(cancellationToken); - await command.ExecuteNonQueryAsync(cancellationToken); - } - catch (SqlException sqlException) - { - if (sqlException.Number == MsSqlDuplicateKeyError_UniqueIndexViolation || - sqlException.Number == MsSqlDuplicateKeyError_UniqueConstraintViolation) - { - loggingAction?.Invoke(); - return; - } - - throw; - } - finally - { - FinishWrite(connection, transactionProvider); - } - } - - protected override T ReadFromStore( - Func commandFunc, - Func resultFunc - ) - { - var connection = _connectionProvider.GetConnection(); - - if (connection.State != ConnectionState.Open) - connection.Open(); - using var command = commandFunc.Invoke(connection); - try - { - return resultFunc.Invoke(command.ExecuteReader()); - } - finally - { - connection.Close(); - } - } - - protected override async Task ReadFromStoreAsync( - Func commandFunc, - Func> resultFunc, - CancellationToken cancellationToken - ) - { - var connection = await _connectionProvider.GetConnectionAsync(cancellationToken); - - if (connection.State != ConnectionState.Open) - await connection.OpenAsync(cancellationToken); - using var command = commandFunc.Invoke(connection); - try - { - return await resultFunc.Invoke(await command.ExecuteReaderAsync(cancellationToken)); - } - finally - { - connection.Close(); - } - } - - protected override DbCommand CreateCommand( - DbConnection connection, - string sqlText, - int outBoxTimeout, - params IDbDataParameter[] parameters - ) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan since, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = new SqlParameter { ParameterName = "PageNumber", Value = pageNumber }; - parameters[1] = new SqlParameter { ParameterName = "PageSize", Value = pageSize }; - parameters[2] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(since)); - return parameters; - } - - protected override IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan dispatchedSince, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = new SqlParameter { ParameterName = "PageNumber", Value = pageNumber }; - parameters[1] = new SqlParameter { ParameterName = "PageSize", Value = pageSize }; - parameters[2] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); - - return parameters; - } - - protected override IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber) - { - var parameters = new IDbDataParameter[2]; - parameters[0] = new SqlParameter { ParameterName = "PageNumber", Value = pageNumber }; - parameters[1] = new SqlParameter { ParameterName = "PageSize", Value = pageSize }; - return parameters; - } - - #region Parameter Helpers - - protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) - { - return new SqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; - } - - protected override IDbDataParameter[] InitAddDbParameters(Message message, int? position = null) - { - var prefix = position.HasValue ? $"p{position}_" : ""; - - return - [ - new SqlParameter - { - ParameterName = $"{prefix}MessageId", - DbType = DbType.String, - Value = message.Id.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}MessageType", - DbType = DbType.String, - Value = message.Header.MessageType.ToString() - }, - new SqlParameter - { - ParameterName = $"{prefix}Topic", - DbType = DbType.String, - Value = message.Header.Topic.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}Timestamp", - DbType = DbType.DateTimeOffset, - Value = message.Header.TimeStamp.ToUniversalTime() - }, - new SqlParameter - { - ParameterName = $"{prefix}CorrelationId", - DbType = DbType.String, - Value = message.Header.CorrelationId.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}ReplyTo", - DbType = DbType.String, - Value = message.Header.ReplyTo is not null ? message.Header.ReplyTo.Value : DBNull.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}ContentType", - DbType = DbType.String, - Value = message.Header.ContentType is not null ? message.Header.ContentType.ToString() : DBNull.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}PartitionKey", - DbType = DbType.String, - Value = message.Header.PartitionKey.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}Source", - DbType = DbType.String, - Value = message.Header.Source.AbsoluteUri - }, - new SqlParameter - { - ParameterName = $"{prefix}Type", - DbType = DbType.String, - Value = message.Header.Type - }, - new SqlParameter - { - ParameterName = $"{prefix}DataSchema", - DbType = DbType.String, - Value = message.Header.DataSchema is not null ? message.Header.DataSchema.AbsoluteUri : DBNull.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}Subject", - DbType = DbType.String, - Value = message.Header.Subject is not null ? message.Header.Subject : DBNull.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}SpecVersion", - DbType = DbType.String, - Value = message.Header.SpecVersion - }, - new SqlParameter - { - ParameterName = $"{prefix}HandledCount", - DbType = DbType.Int32, - Value = message.Header.HandledCount - }, - new SqlParameter - { - ParameterName = $"{prefix}Delayed", - DbType = DbType.Int64, - Value = message.Header.Delayed.Ticks - }, - new SqlParameter - { - ParameterName = $"{prefix}TraceParent", - DbType = DbType.String, - Value = message.Header.TraceParent is not null ? message.Header.TraceParent.Value : DBNull.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}TraceState", - DbType = DbType.String, - Value = message.Header.TraceState is not null ? message.Header.TraceState.Value : DBNull.Value - }, - new SqlParameter - { - ParameterName = $"{prefix}Baggage", - DbType = DbType.String, - Value = message.Header.Baggage.ToString() - }, - new SqlParameter - { - ParameterName = $"{prefix}DataRef", - DbType = DbType.String, - Value = message.Header.DataRef is not null ? message.Header.DataRef : DBNull.Value - }, - // Bag as JSON - new SqlParameter - { - ParameterName = $"{prefix}HeaderBag", - Value = JsonSerializer.Serialize(message.Header.Bag, JsonSerialisationOptions.Options) - }, - _configuration.BinaryMessagePayload - ? new SqlParameter - { - ParameterName = $"{prefix}Body", - DbType = DbType.Binary, - Value = message.Body.Bytes - } - : new SqlParameter - { - ParameterName = $"{prefix}Body", - DbType = DbType.String, - Value = message.Body.Value - } - ]; - } - - #endregion - - #region Property Extractors - - private static Baggage GetBaggage(DbDataReader dr) - { - var (i, err) = TryGetOrdinal(dr, "Baggage"); - if (err || dr.IsDBNull(i)) - return new Baggage(); // If the column does not exist or is null, return an empty Baggage - - var baggageString = dr.IsDBNull(i) ? string.Empty: dr.GetString(i); - - var baggage = new Baggage(); - baggage.LoadBaggage(baggageString); - return baggage; - } - - private static byte[]? GetBodyAsBytes(SqlDataReader dr) - { - var ordinal = dr.GetOrdinal("Body"); - if (dr.IsDBNull(ordinal)) return null; - - var body = dr.GetStream(ordinal); - if (body is MemoryStream memoryStream) // No need to dispose a MemoryStream, I do not think they dare to ever change that - return memoryStream.ToArray(); // Then we can just return its value, instead of copying manually - - MemoryStream ms = new(); - body.CopyTo(ms); - body.Dispose(); - return ms.ToArray(); - } - - private static string? GetBodyAsText(DbDataReader dr) - { - var ordinal = dr.GetOrdinal("Body"); - return dr.IsDBNull(ordinal) ? null : dr.GetString(ordinal); - } - - private static Dictionary? GetContextBag(DbDataReader dr) - { - var i = dr.GetOrdinal("HeaderBag"); - var headerBag = dr.IsDBNull(i) ? "" : dr.GetString(i); - var dictionaryBag = - JsonSerializer.Deserialize>(headerBag, JsonSerialisationOptions.Options); - return dictionaryBag; - } - - private static string? GetCorrelationId(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "CorrelationId"); - if (err || dr.IsDBNull(ordinal)) return null; // If the column does not exist or is null, return null - - var correlationId = dr.GetString(ordinal); - return correlationId; - } - - private static string? GetDataRef(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "DataRef"); - if (err || dr.IsDBNull(ordinal)) return null; - - var dataRef = dr.GetString(ordinal); - return string.IsNullOrEmpty(dataRef) ? null : dataRef; - } - - private static Uri? GetDataSchema(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "DataSchema"); - if (err || dr.IsDBNull(ordinal)) return null; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? null : new Uri(source); - } - - private static string GetMessageId(DbDataReader dr) => dr.GetString(dr.GetOrdinal("MessageId")); - - private static string? GetContentType(DbDataReader dr) - { - var ordinal = dr.GetOrdinal("ContentType"); - if (dr.IsDBNull(ordinal)) return null; - - var contentType = dr.GetString(ordinal); - return contentType; - } - - private static MessageType GetMessageType(DbDataReader dr) => - (MessageType)Enum.Parse(typeof(MessageType), dr.GetString(dr.GetOrdinal("MessageType"))); - - private static string? GetPartitionKey(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "PartitionKey"); - if (err || dr.IsDBNull(ordinal)) return null; - - var partitionKey = dr.GetString(ordinal); - return partitionKey; - } - - private static string? GetReplyTo(DbDataReader dr) - { - var ordinal = dr.GetOrdinal("ReplyTo"); - if (dr.IsDBNull(ordinal)) return null; - - var replyTo = dr.GetString(ordinal); - return replyTo; - } - - private static string? GetSpecVersion(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "SpecVersion"); - if (err || dr.IsDBNull(ordinal)) return null;; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? null : source; - } - - private static Uri? GetSource(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Source"); - if (err || dr.IsDBNull(ordinal)) return null; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? null : new Uri(source); - } - - private static string? GetSubject(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Subject"); - if (err || dr.IsDBNull(ordinal)) return null; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? null : source; - } - - private static DateTimeOffset GetTimeStamp(DbDataReader dr) - { - var ordinal = dr.GetOrdinal("Timestamp"); - var timeStamp = dr.IsDBNull(ordinal) - ? DateTimeOffset.MinValue - : dr.GetDateTime(ordinal); - return timeStamp; - } - - private static RoutingKey GetTopic(DbDataReader dr) => new RoutingKey(dr.GetString(dr.GetOrdinal("Topic"))); - - private static TraceParent GetTraceParent(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "TraceParent"); - if (err || dr.IsDBNull(ordinal)) return TraceParent.Empty; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? TraceParent.Empty : new TraceParent(source); - } - - private static TraceState GetTraceState(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "TraceState"); - if (dr.IsDBNull(ordinal)) return TraceState.Empty; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? TraceState.Empty : new TraceState(source); - } - - private static string? GetType(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Type"); - if (dr.IsDBNull(ordinal)) return null; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? null : source; - } - - private static (int, bool) TryGetOrdinal(DbDataReader dr, string columnName) - { - try - { - return (dr.GetOrdinal(columnName), false); - } - catch (IndexOutOfRangeException) - { - // SpecVersion column does not exist, return -1 and true to indicate error - return (-1, true); - } - } - - #endregion - - #region DataReader Operators - - protected override Message MapFunction(DbDataReader dr) - { - Message? message = null; - if (dr.Read()) - { - message = MapAMessage(dr); - } - - dr.Close(); - return message ?? new Message(); - } - - protected override async Task MapFunctionAsync(DbDataReader dr, CancellationToken cancellationToken) - { - Message? message = null; - if (await dr.ReadAsync(cancellationToken)) - { - message = MapAMessage(dr); - } - -#if NET462 - dr.Close(); -#else - await dr.CloseAsync(); -#endif - return message ?? new Message(); - } - - protected override IEnumerable MapListFunction(DbDataReader dr) - { - var messages = new List(); - while (dr.Read()) - { - messages.Add(MapAMessage(dr)); - } - - dr.Close(); - - return messages; - } - - protected override async Task> MapListFunctionAsync( - DbDataReader dr, - CancellationToken cancellationToken - ) - { - var messages = new List(); - while (await dr.ReadAsync(cancellationToken)) - { - messages.Add(MapAMessage(dr)); - } + } -#if NET462 - dr.Close(); -#else - await dr.CloseAsync(); -#endif - return messages; - } + /// + /// Initializes a new instance of the class. + /// + /// The configuration. + public MsSqlOutbox(IAmARelationalDatabaseConfiguration configuration) : this(configuration, + new MsSqlConnectionProvider(configuration)) + { + } - protected override async Task MapOutstandingCountAsync(DbDataReader dr, - CancellationToken cancellationToken) - { - int outstandingMessages = -1; - if (await dr.ReadAsync(cancellationToken)) - { - outstandingMessages = dr.GetInt32(0); - } + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) + { + return ex is SqlException { Number: MsSqlDuplicateKeyError_UniqueIndexViolation or MsSqlDuplicateKeyError_UniqueConstraintViolation }; + } -#if NET462 - dr.Close(); -#else - await dr.CloseAsync(); -#endif - return outstandingMessages; - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) + { + return new SqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; + } - protected override int MapOutstandingCount(DbDataReader dr) + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, DbType dbType, object? value) + { + return new SqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value, DbType = dbType }; + } + + /// + protected override DateTimeOffset GetTimeStamp(DbDataReader dr) + { + if (!TryGetOrdinal(dr, TimestampColumnName, out var ordinal) || dr.IsDBNull(ordinal)) { - int outstandingMessages = -1; - if (dr.Read()) - { - outstandingMessages = dr.GetInt32(0); - } - - dr.Close(); - return outstandingMessages; + return DateTimeOffset.MinValue; } - #endregion - - private Message MapAMessage(DbDataReader dr) - { - var id = GetMessageId(dr); - var messageType = GetMessageType(dr); - var topic = GetTopic(dr); - - DateTimeOffset timeStamp = GetTimeStamp(dr); - var correlationId = GetCorrelationId(dr); - var replyTo = GetReplyTo(dr); - var contentType = GetContentType(dr); - var partitionKey = GetPartitionKey(dr); - - var source = GetSource(dr); - var type = GetType(dr); - var dataSchema = GetDataSchema(dr); - var subject = GetSubject(dr); - var specVersion = GetSpecVersion(dr); - var traceParent = GetTraceParent(dr); - var traceState = GetTraceState(dr); - var baggage = GetBaggage(dr); - var dataRef = GetDataRef(dr); - - var header = new MessageHeader( - messageId: new Id(id), - topic: topic, - messageType: messageType, - source: source, - type: type, - timeStamp: timeStamp, - correlationId: correlationId is not null ? new Id(correlationId) : Id.Empty, - replyTo: replyTo is not null ? new RoutingKey(replyTo) : RoutingKey.Empty, - contentType: contentType is not null ? new ContentType(contentType) : new ContentType(MediaTypeNames.Text.Plain), - partitionKey: partitionKey is not null ? new PartitionKey(partitionKey) : PartitionKey.Empty, - dataSchema: dataSchema, - subject: subject, - handledCount: 0, // HandledCount is zero when restored from the Outbox - delayed: TimeSpan.Zero, // Delayed is zero when restored from the Outbox - traceParent: traceParent, - traceState: traceState, - baggage: baggage - ); - header.SpecVersion = specVersion ?? MessageHeader.DefaultSpecVersion; - header.DataRef = dataRef; - - Dictionary? dictionaryBag = GetContextBag(dr); - if (dictionaryBag != null) - { - foreach (var keyValue in dictionaryBag) - { - header.Bag.Add(keyValue.Key, keyValue.Value); - } - } - -#if NET462 - var body = _configuration.BinaryMessagePayload - ? new MessageBody(GetBodyAsBytes((SqlDataReader)dr), new ContentType(MediaTypeNames.Application.Octet), CharacterEncoding.Raw) - : new MessageBody(GetBodyAsText(dr), new ContentType("applicaton/json"), CharacterEncoding.UTF8); -#else - var body = _configuration.BinaryMessagePayload - ? new MessageBody(GetBodyAsBytes((SqlDataReader)dr), new ContentType(MediaTypeNames.Application.Octet), CharacterEncoding.Raw) - : new MessageBody(GetBodyAsText(dr), new ContentType(MediaTypeNames.Application.Json), CharacterEncoding.UTF8); -#endif - return new Message(header, body); - } + var reader = (SqlDataReader)dr; + var dataTime = reader.GetDateTimeOffset(ordinal); + return dataTime; } } diff --git a/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs b/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs index c5b97376b1..b9fbe7d34b 100644 --- a/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs +++ b/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs @@ -24,17 +24,9 @@ THE SOFTWARE. */ #endregion using System; -using System.Collections.Generic; using System.Data; using System.Data.Common; -using System.IO; -using System.Net.Mime; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; using MySqlConnector; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.MySql; using Paramore.Brighter.Observability; @@ -44,13 +36,9 @@ namespace Paramore.Brighter.Outbox.MySql /// /// Implements an outbox using Sqlite as a backing store /// - public partial class MySqlOutbox : RelationDatabaseOutbox + public class MySqlOutbox : RelationDatabaseOutbox { - private static readonly ILogger s_logger = ApplicationLogging.CreateLogger(); - private const int MySqlDuplicateKeyError = 1062; - private readonly IAmARelationalDatabaseConfiguration _configuration; - private readonly IAmARelationalDbConnectionProvider _connectionProvider; /// /// Initializes a new instance of the class. @@ -59,12 +47,9 @@ public partial class MySqlOutbox : RelationDatabaseOutbox /// Provides a connection to the Db that allows us to enlist in an ambient transaction public MySqlOutbox(IAmARelationalDatabaseConfiguration configuration, IAmARelationalDbConnectionProvider connectionProvider) - : base(DbSystem.MySql, configuration.DatabaseName, configuration.OutBoxTableName, + : base(DbSystem.MySql, configuration, connectionProvider, new MySqlQueries(), ApplicationLogging.CreateLogger()) { - _configuration = configuration; - _connectionProvider = connectionProvider; - ContinueOnCapturedContext = false; } /// @@ -76,489 +61,35 @@ public MySqlOutbox(IAmARelationalDatabaseConfiguration configuration) { } - protected override void WriteToStore( - IAmABoxTransactionProvider transactionProvider, - Func commandFunc, - Action loggingAction - ) - { - var connection = GetOpenConnection(_connectionProvider, transactionProvider); - using var command = commandFunc.Invoke(connection); - try - { - if (transactionProvider is { HasOpenTransaction: true }) - command.Transaction = transactionProvider.GetTransaction(); - command.ExecuteNonQuery(); - } - catch (MySqlException sqlException) - { - if (!IsExceptionUnqiueOrDuplicateIssue(sqlException)) throw; - Log.DuplicateDetectedInBatch(s_logger); - } - finally - { - FinishWrite(connection, transactionProvider); - } - } - - protected override async Task WriteToStoreAsync( - IAmABoxTransactionProvider transactionProvider, - Func commandFunc, - Action loggingAction, - CancellationToken cancellationToken - ) - { - var connection = await GetOpenConnectionAsync(_connectionProvider, transactionProvider, cancellationToken); - using var command = commandFunc.Invoke(connection); - try - { - if (transactionProvider is { HasOpenTransaction: true }) - command.Transaction = await transactionProvider.GetTransactionAsync(cancellationToken); - await command.ExecuteNonQueryAsync(cancellationToken); - } - catch (MySqlException sqlException) - { - if (IsExceptionUnqiueOrDuplicateIssue(sqlException)) - { - Log.DuplicateDetectedInBatch(s_logger); - return; - } - - throw; - } - finally - { - FinishWrite(connection, transactionProvider); - } - } - - protected override T ReadFromStore( - Func commandFunc, - Func resultFunc - ) - { - var connection = _connectionProvider.GetConnection(); - - if (connection.State != ConnectionState.Open) - connection.Open(); - using var command = commandFunc.Invoke(connection); - try - { - return resultFunc.Invoke(command.ExecuteReader()); - } - finally - { - connection.Close(); - } - } - - protected override async Task ReadFromStoreAsync( - Func commandFunc, - Func> resultFunc, - CancellationToken cancellationToken) - { - var connection = await _connectionProvider.GetConnectionAsync(cancellationToken); - - if (connection.State != ConnectionState.Open) - await connection.OpenAsync(cancellationToken); -#if NETSTANDARD2_0 - using var command = commandFunc.Invoke(connection); -#else - await using var command = commandFunc.Invoke(connection); -#endif - try - { - return await resultFunc.Invoke(await command.ExecuteReaderAsync(cancellationToken)); - } - finally - { -#if NETSTANDARD2_0 - connection.Close(); -#else - await connection.CloseAsync(); -#endif - } - } - - protected override DbCommand CreateCommand( - DbConnection connection, - string sqlText, - int outBoxTimeout, - params IDbDataParameter[] parameters) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan dispatchedSince, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); - - return parameters; - } - - protected override IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber) - { - var parameters = new IDbDataParameter[2]; - parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - parameters[1] = CreateSqlParameter("Take", pageSize); - - return parameters; - } - - protected override IDbDataParameter CreateSqlParameter(string parameterName, object value) - { - return new MySqlParameter { ParameterName = parameterName, Value = value }; - } - - - protected override IDbDataParameter[] InitAddDbParameters(Message message, int? position = null) - { - var prefix = position.HasValue ? $"p{position}_" : ""; - var bagJson = JsonSerializer.Serialize(message.Header.Bag, JsonSerialisationOptions.Options); - - return new[] - { - new MySqlParameter { ParameterName = $"@{prefix}MessageId", DbType = DbType.String, Value = message.Id.Value }, - new MySqlParameter { ParameterName = $"@{prefix}MessageType", DbType = DbType.String, Value = message.Header.MessageType.ToString() }, - new MySqlParameter { ParameterName = $"@{prefix}Topic", DbType = DbType.String, Value = message.Header.Topic.Value, }, - new MySqlParameter - { - ParameterName = $"@{prefix}Timestamp", DbType = DbType.DateTimeOffset, Value = message.Header.TimeStamp.ToUniversalTime() - }, //always store in UTC, as this is how we query messages - new MySqlParameter { ParameterName = $"@{prefix}CorrelationId", DbType = DbType.String, Value = message.Header.CorrelationId.Value }, - new MySqlParameter { ParameterName = $"@{prefix}ReplyTo", DbType = DbType.String, Value = message.Header.ReplyTo?.Value }, - new MySqlParameter { ParameterName = $"@{prefix}ContentType", DbType = DbType.String, Value = message.Header.ContentType?.ToString() }, - new MySqlParameter { ParameterName = $"@{prefix}PartitionKey", DbType = DbType.String, Value = message.Header.PartitionKey.Value }, - new MySqlParameter { ParameterName = $"@{prefix}HeaderBag", DbType = DbType.String, Value = bagJson }, _configuration.BinaryMessagePayload - ? new MySqlParameter { ParameterName = $"@{prefix}Body", DbType = DbType.Binary, Value = message.Body.Bytes } - : new MySqlParameter { ParameterName = $"@{prefix}Body", DbType = DbType.String, Value = message.Body.Value }, - new MySqlParameter { ParameterName = $"@{prefix}Source", DbType = DbType.String, Value = message.Header.Source.AbsoluteUri }, - new MySqlParameter { ParameterName = $"@{prefix}Type", DbType = DbType.String, Value = message.Header.Type }, - new MySqlParameter { ParameterName = $"@{prefix}DataSchema", DbType = DbType.String, Value = message.Header.DataSchema?.AbsoluteUri }, - new MySqlParameter { ParameterName = $"@{prefix}Subject", DbType = DbType.String, Value = message.Header.Subject }, - new MySqlParameter { ParameterName = $"@{prefix}TraceParent", DbType = DbType.String, Value = message.Header.TraceParent?.Value }, - new MySqlParameter { ParameterName = $"@{prefix}TraceState", DbType = DbType.String, Value = message.Header.TraceState?.Value }, - new MySqlParameter { ParameterName = $"@{prefix}Baggage", DbType = DbType.String, Value = message.Header.Baggage.ToString() } - }; - } - - protected override IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan since, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("TimestampSince", DateTimeOffset.UtcNow.Subtract(since)); - - return parameters; - } - - protected override Message MapFunction(DbDataReader dr) - { - if (dr.Read()) - { - return MapAMessage(dr); - } - - return new Message(); - } - - protected override async Task MapFunctionAsync(DbDataReader dr, CancellationToken cancellationToken) - { - if (await dr.ReadAsync(cancellationToken)) - { - return MapAMessage(dr); - } - - return new Message(); - } - - protected override IEnumerable MapListFunction(DbDataReader dr) - { - var messages = new List(); - while (dr.Read()) - { - messages.Add(MapAMessage(dr)); - } - - dr.Close(); - - return messages; - } - - protected override async Task> MapListFunctionAsync( - DbDataReader dr, - CancellationToken cancellationToken) - { - var messages = new List(); - while (await dr.ReadAsync(cancellationToken)) - { - messages.Add(MapAMessage(dr)); - } -#if NETSTANDARD2_0 - dr.Close(); -#else - await dr.CloseAsync(); -#endif - - return messages; - } - - - protected override async Task MapOutstandingCountAsync(DbDataReader dr, - CancellationToken cancellationToken) + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) { - int outstandingMessages = -1; - if (await dr.ReadAsync(cancellationToken)) - { - outstandingMessages = dr.GetInt32(0); - } - -#if NETSTANDARD2_0 - dr.Close(); -#else - await dr.CloseAsync(); -#endif - - return outstandingMessages; + return new MySqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; } - protected override int MapOutstandingCount(DbDataReader dr) + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, DbType dbType, object? value) { - int outstandingMessages = -1; - if (dr.Read()) - { - outstandingMessages = dr.GetInt32(0); - } - - dr.Close(); - return outstandingMessages; + return new MySqlParameter { ParameterName = parameterName, Value = value, DbType = dbType }; } - private static bool IsExceptionUnqiueOrDuplicateIssue(MySqlException sqlException) + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) { - return sqlException.Number == MySqlDuplicateKeyError; + return ex is MySqlException { Number: MySqlDuplicateKeyError }; } - - private Message MapAMessage(DbDataReader dr) + + /// + protected override DateTimeOffset GetTimeStamp(DbDataReader dr) { - var id = GetMessageId(dr); - var messageType = GetMessageType(dr); - var topic = GetTopic(dr); - - var header = new MessageHeader(id, topic, messageType); - - if (dr.FieldCount > 4) + if (!TryGetOrdinal(dr, TimestampColumnName, out var ordinal) || dr.IsDBNull(ordinal)) { - var timeStamp = GetTimeStamp(dr); - var correlationId = GetCorrelationId(dr); - var replyTo = GetReplyTo(dr); - var contentType = GetContentType(dr) ?? new ContentType(MediaTypeNames.Text.Plain); - var partitionKey = GetPartitionKey(dr); - var source = GetSource(dr); - var eventType = GetEventType(dr); - var dataSchema = GetDataSchema(dr); - var subject = GetSubject(dr); - var traceParent = GetTraceParent(dr); - var traceState = GetTraceState(dr); - var baggage = GetBaggage(dr); - - header = new MessageHeader( - messageId: id, - topic: topic, - messageType: messageType, - source: source, - type: eventType, - timeStamp: timeStamp, - correlationId: correlationId, - replyTo: replyTo, - contentType: contentType, - partitionKey: partitionKey, - dataSchema: dataSchema, - subject: subject, - handledCount: 0, - delayed: TimeSpan.Zero, - traceParent: traceParent, - traceState: traceState, - baggage: baggage - ); - - // existing bag items - var dictionaryBag = GetContextBag(dr); - if (dictionaryBag != null) - foreach (var kv in dictionaryBag) - header.Bag.Add(kv.Key, kv.Value); + return DateTimeOffset.MinValue; } -#if NETSTANDARD2_0 - var body = _configuration.BinaryMessagePayload - ? new MessageBody(GetBodyAsBytes((MySqlDataReader)dr), new ContentType(MediaTypeNames.Application.Octet), CharacterEncoding.Raw) - : new MessageBody(GetBodyAsString(dr), new ContentType("application/json"), CharacterEncoding.UTF8); - -#else - var body = _configuration.BinaryMessagePayload - ? new MessageBody(GetBodyAsBytes((MySqlDataReader)dr), new ContentType(MediaTypeNames.Application.Octet), CharacterEncoding.Raw) - : new MessageBody(GetBodyAsString(dr), new ContentType(MediaTypeNames.Application.Json), CharacterEncoding.UTF8); - -#endif - - return new Message(header, body); - } - - private static byte[] GetBodyAsBytes(MySqlDataReader dr) - { - // No need to dispose a MemoryStream, I do not think they dare to ever change that - var stream = dr.GetStream("Body"); - if (stream is not MemoryStream memoryStream) // the current implementation returns a MemoryStream - // If the type of returned Stream is ever changed, please check if it requires disposal, also other places in the code base that uses GetStream - throw new NotImplementedException(nameof(MySqlDataReader.GetStream) + " no longer returns " + nameof(MemoryStream)); - - return memoryStream.ToArray(); // Then we can just return its value, instead of copying manually - } - - private static string GetBodyAsString(IDataReader dr) - { - return dr.GetString(dr.GetOrdinal("Body")); - } - - private static Dictionary GetContextBag(IDataReader dr) - { - var i = dr.GetOrdinal("HeaderBag"); - var headerBag = dr.IsDBNull(i) ? "" : dr.GetString(i); - var dictionaryBag = - JsonSerializer.Deserialize>(headerBag, JsonSerialisationOptions.Options); - return dictionaryBag; - } - - private static ContentType GetContentType(IDataReader dr) - { - var ordinal = dr.GetOrdinal("ContentType"); - if (dr.IsDBNull(ordinal)) return null; - - var contentType = dr.GetString(ordinal); - if (string.IsNullOrEmpty(contentType)) - return null; - return new ContentType(contentType); - } - - private static Id GetCorrelationId(IDataReader dr) - { - var ordinal = dr.GetOrdinal("CorrelationId"); - if (dr.IsDBNull(ordinal)) return null; - - var correlationId = dr.GetString(ordinal); - return new Id(correlationId); - } - - private static MessageType GetMessageType(IDataReader dr) - { - return (MessageType)Enum.Parse(typeof(MessageType), dr.GetString(dr.GetOrdinal("MessageType"))); - } - - private static Id GetMessageId(IDataReader dr) - { - var id = dr.GetString(dr.GetOrdinal("MessageId")); - return new Id(id); - } - - private static string GetPartitionKey(IDataReader dr) - { - var ordinal = dr.GetOrdinal("PartitionKey"); - if (dr.IsDBNull(ordinal)) return null; - - var partitionKey = dr.GetString(ordinal); - return partitionKey; - } - - private static RoutingKey GetReplyTo(IDataReader dr) - { - var ordinal = dr.GetOrdinal("ReplyTo"); - if (dr.IsDBNull(ordinal)) return null; - - var replyTo = dr.GetString(ordinal); - if (string.IsNullOrEmpty(replyTo)) - return null; - return new RoutingKey(replyTo); - } - - private static RoutingKey GetTopic(IDataReader dr) - { - return new RoutingKey(dr.GetString(dr.GetOrdinal("Topic"))); - } - - private static DateTimeOffset GetTimeStamp(IDataReader dr) - { - var ordinal = dr.GetOrdinal("Timestamp"); - var timeStamp = dr.IsDBNull(ordinal) - ? DateTimeOffset.MinValue - : dr.GetDateTime(ordinal); - return timeStamp; - } - - private static Uri GetSource(IDataReader dr) - { - var ord = dr.GetOrdinal("Source"); - return dr.IsDBNull(ord) ? null : new Uri(dr.GetString(ord)); - } - - private static string GetEventType(IDataReader dr) - { - var ord = dr.GetOrdinal("Type"); - return dr.IsDBNull(ord) ? null : dr.GetString(ord); - } - - private static Uri GetDataSchema(IDataReader dr) - { - var ord = dr.GetOrdinal("DataSchema"); - return dr.IsDBNull(ord) ? null : new Uri(dr.GetString(ord)); - } - - private static string GetSubject(IDataReader dr) - { - var ord = dr.GetOrdinal("Subject"); - return dr.IsDBNull(ord) ? null : dr.GetString(ord); - } - - private static TraceParent GetTraceParent(IDataReader dr) - { - var ord = dr.GetOrdinal("TraceParent"); - return dr.IsDBNull(ord) ? null : new TraceParent(dr.GetString(ord)); - } - - private static TraceState GetTraceState(IDataReader dr) - { - var ord = dr.GetOrdinal("TraceState"); - return dr.IsDBNull(ord) ? null : new TraceState(dr.GetString(ord)); - } - - private static Baggage GetBaggage(IDataReader dr) - { - var baggage = new Baggage(); - - var ord = dr.GetOrdinal("Baggage"); - - var baggageAsString = dr.IsDBNull(ord) ? "" : dr.GetString(ord); - if (string.IsNullOrEmpty(baggageAsString)) - return baggage; - - baggage.LoadBaggage(baggageAsString); - return baggage; - } - - private static partial class Log - { - [LoggerMessage(LogLevel.Warning, "MsSqlOutbox: A duplicate was detected in the batch")] - public static partial void DuplicateDetectedInBatch(ILogger logger); + var reader = (MySqlDataReader)dr; + var dataTime = reader.GetDateTimeOffset(ordinal); + return dataTime; } } } diff --git a/src/Paramore.Brighter.Outbox.MySql/Paramore.Brighter.Outbox.MySql.csproj b/src/Paramore.Brighter.Outbox.MySql/Paramore.Brighter.Outbox.MySql.csproj index 68f4fc1ef6..69be4e961f 100644 --- a/src/Paramore.Brighter.Outbox.MySql/Paramore.Brighter.Outbox.MySql.csproj +++ b/src/Paramore.Brighter.Outbox.MySql/Paramore.Brighter.Outbox.MySql.csproj @@ -4,6 +4,7 @@ Derek Comartin $(BrighterTargetFrameworks) RabbitMQ;AMQP;Command;Event;Service Activator;Decoupled;Invocation;Messaging;Remote;Command Dispatcher;Command Processor;Request;Service;Task Queue;Work Queue;Retry;Circuit Breaker;Availability + enable diff --git a/src/Paramore.Brighter.Outbox.PostgreSql/PostgreSqlOutbox.cs b/src/Paramore.Brighter.Outbox.PostgreSql/PostgreSqlOutbox.cs index 9cb022d669..25b0857ecd 100644 --- a/src/Paramore.Brighter.Outbox.PostgreSql/PostgreSqlOutbox.cs +++ b/src/Paramore.Brighter.Outbox.PostgreSql/PostgreSqlOutbox.cs @@ -24,678 +24,61 @@ THE SOFTWARE. */ #endregion using System; -using System.Collections.Generic; using System.Data; -using System.Data.Common; -using System.Net.Mime; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; using Npgsql; -using NpgsqlTypes; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.Observability; using Paramore.Brighter.PostgreSql; -namespace Paramore.Brighter.Outbox.PostgreSql +namespace Paramore.Brighter.Outbox.PostgreSql; + +/// +/// Implements an outbox using PostgreSQL as a backing store +/// +public class PostgreSqlOutbox : RelationDatabaseOutbox { /// - /// Implements an outbox using PostgreSQL as a backing store + /// Initializes a new instance of the class. /// - public partial class PostgreSqlOutbox : RelationDatabaseOutbox + /// The configuration to connect to this data store + /// Provides a connection to the Db that allows us to enlist in an ambient transaction + public PostgreSqlOutbox( + IAmARelationalDatabaseConfiguration configuration, + IAmARelationalDbConnectionProvider connectionProvider) + : base(DbSystem.Postgresql, configuration, connectionProvider, + new PostgreSqlQueries(), ApplicationLogging.CreateLogger()) { - private static readonly ILogger s_logger = ApplicationLogging.CreateLogger(); - - private readonly IAmARelationalDatabaseConfiguration _configuration; - private readonly IAmARelationalDbConnectionProvider _connectionProvider; - - /// - /// Initializes a new instance of the class. - /// - /// The configuration to connect to this data store - /// Provides a connection to the Db that allows us to enlist in an ambient transaction - public PostgreSqlOutbox( - IAmARelationalDatabaseConfiguration configuration, - IAmARelationalDbConnectionProvider connectionProvider) : base( - DbSystem.Postgresql, configuration.DatabaseName, configuration.OutBoxTableName, - new PostgreSqlQueries(), ApplicationLogging.CreateLogger()) - { - _configuration = configuration; - _connectionProvider = connectionProvider; - } - - /// - /// Initializes a new instance of the class. - /// - /// The configuration to connect to this data store - /// From v7.0 Npgsql uses an Npgsql data source, leave null to have Brighter manage - /// connections; Brighter will not manage type mapping for you in this case so you must register them - /// globally - public PostgreSqlOutbox( - IAmARelationalDatabaseConfiguration configuration, - NpgsqlDataSource? dataSource = null) - : this(configuration, new PostgreSqlConnectionProvider(configuration, dataSource)) - { - } - - protected override void WriteToStore( - IAmABoxTransactionProvider? transactionProvider, - Func commandFunc, - Action? loggingAction) - { - var connection = GetOpenConnection(_connectionProvider, transactionProvider); - using var command = commandFunc.Invoke(connection); - try - { - if (transactionProvider is { HasOpenTransaction: true }) - command.Transaction = transactionProvider.GetTransaction(); - command.ExecuteNonQuery(); - } - catch (PostgresException sqlException) - { - if (sqlException.SqlState == PostgresErrorCodes.UniqueViolation) - { - loggingAction?.Invoke(); - return; - } - - throw; - } - finally - { - FinishWrite(connection, transactionProvider); - } - } - - protected override async Task WriteToStoreAsync( - IAmABoxTransactionProvider? transactionProvider, - Func commandFunc, - Action? loggingAction, - CancellationToken cancellationToken) - { - var connection = await GetOpenConnectionAsync(_connectionProvider, transactionProvider, cancellationToken); - await using var command = commandFunc.Invoke(connection); - try - { - if (transactionProvider is { HasOpenTransaction: true }) - command.Transaction = await transactionProvider.GetTransactionAsync(cancellationToken); - await command.ExecuteNonQueryAsync(cancellationToken); - } - catch (PostgresException sqlException) - { - if (sqlException.SqlState == PostgresErrorCodes.UniqueViolation) - { - Log.DuplicateDetectedInBatch(s_logger); - return; - } - - throw; - } - finally - { - FinishWrite(connection, transactionProvider); - } - } - - protected override T ReadFromStore( - Func commandFunc, - Func resultFunc - ) - { - var connection = _connectionProvider.GetConnection(); - - if (connection.State != ConnectionState.Open) - connection.Open(); - using var command = commandFunc.Invoke(connection); - try - { - return resultFunc.Invoke(command.ExecuteReader()); - } - finally - { - connection.Close(); - } - } - - protected override async Task ReadFromStoreAsync( - Func commandFunc, - Func> resultFunc, - CancellationToken cancellationToken - ) - { - var connection = await _connectionProvider.GetConnectionAsync(cancellationToken); - - if (connection.State != ConnectionState.Open) - await connection.OpenAsync(cancellationToken); - await using var command = commandFunc.Invoke(connection); - try - { - return await resultFunc.Invoke(await command.ExecuteReaderAsync(cancellationToken)); - } - finally - { - await connection.CloseAsync(); - } - } - - protected override DbCommand CreateCommand( - DbConnection connection, - string sqlText, - int outBoxTimeout, - params IDbDataParameter[] parameters) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan since, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("TimestampSince", DateTimeOffset.UtcNow.Subtract(since)); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - - return parameters; - } - - protected override IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan dispatchedSince, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - - return parameters; - } - - protected override IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber) - { - var parameters = new IDbDataParameter[2]; - parameters[0] = CreateSqlParameter("Take", pageSize); - parameters[1] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - - return parameters; - } - - protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) - { - return new NpgsqlParameter { ParameterName = parameterName, Value = value }; - } - - protected override IDbDataParameter[] InitAddDbParameters(Message message, int? position = null) - { - var prefix = position.HasValue ? $"p{position}_" : ""; - var bagjson = JsonSerializer.Serialize(message.Header.Bag, JsonSerialisationOptions.Options); - return new[] - { - new NpgsqlParameter - { - ParameterName = $"{prefix}MessageId", - NpgsqlDbType = NpgsqlDbType.Text, - Value = message.Id.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}MessageType", - NpgsqlDbType = NpgsqlDbType.Text, - Value = message.Header.MessageType.ToString() - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}Topic", - NpgsqlDbType = NpgsqlDbType.Text, - Value = message.Header.Topic.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}Timestamp", - NpgsqlDbType = NpgsqlDbType.TimestampTz, - Value = message.Header.TimeStamp.ToUniversalTime() - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}CorrelationId", - NpgsqlDbType = NpgsqlDbType.Text, - Value = message.Header.CorrelationId.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}ReplyTo", - NpgsqlDbType = NpgsqlDbType.Varchar, - Value = message.Header.ReplyTo is not null ? message.Header.ReplyTo.Value : DBNull.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}ContentType", - NpgsqlDbType = NpgsqlDbType.Varchar, - Value = message.Header.ContentType is not null ? message.Header.ContentType.ToString() : new ContentType(MediaTypeNames.Text.Plain).ToString() - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}PartitionKey", - NpgsqlDbType = NpgsqlDbType.Varchar, - Value = message.Header.PartitionKey.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}HeaderBag", - NpgsqlDbType = NpgsqlDbType.Text, - Value = bagjson - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}Source", - DbType = DbType.String, - Value = message.Header.Source.AbsoluteUri - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}Type", - DbType = DbType.String, - Value = message.Header.Type - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}DataSchema", - DbType = DbType.String, - Value = message.Header.DataSchema is not null ? message.Header.DataSchema.AbsoluteUri : DBNull.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}Subject", - DbType = DbType.String, - Value = message.Header.Subject is not null ? message.Header.Subject : DBNull.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}SpecVersion", - DbType = DbType.String, - Value = message.Header.SpecVersion - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}TraceParent", - DbType = DbType.String, - Value = message.Header.TraceParent is not null ? message.Header.TraceParent.Value : DBNull.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}TraceState", - DbType = DbType.String, - Value = message.Header.TraceState is not null ? message.Header.TraceState.Value : DBNull.Value - }, - new NpgsqlParameter - { - ParameterName = $"{prefix}Baggage", - DbType = DbType.String, - Value = message.Header.Baggage.ToString() - }, - _configuration.BinaryMessagePayload - ? new NpgsqlParameter - { - ParameterName = $"{prefix}Body", - NpgsqlDbType = NpgsqlDbType.Bytea, - Value = message.Body.Bytes - } - : new NpgsqlParameter - { - ParameterName = $"{prefix}Body", - NpgsqlDbType = NpgsqlDbType.Text, - Value = message.Body.Value - } - }; - } - - protected override Message MapFunction(DbDataReader dr) - { - if (dr.Read()) - { - return MapAMessage(dr); - } - - return new Message(); - } - - protected override async Task MapFunctionAsync(DbDataReader dr, CancellationToken cancellationToken) - { - if (await dr.ReadAsync(cancellationToken)) - { - return MapAMessage(dr); - } - - return new Message(); - } - - protected override IEnumerable MapListFunction(DbDataReader dr) - { - var messages = new List(); - while (dr.Read()) - { - messages.Add(MapAMessage(dr)); - } - - dr.Close(); - - return messages; - } - - protected override async Task> MapListFunctionAsync( - DbDataReader dr, - CancellationToken cancellationToken - ) - { - var messages = new List(); - while (await dr.ReadAsync(cancellationToken)) - { - messages.Add(MapAMessage(dr)); - } - - dr.Close(); - - return messages; - } - - protected override async Task MapOutstandingCountAsync(DbDataReader dr, - CancellationToken cancellationToken) - { - int outstandingMessages = -1; - if (await dr.ReadAsync(cancellationToken)) - { - outstandingMessages = dr.GetInt32(0); - } - - await dr.CloseAsync(); - - return outstandingMessages; - } - - protected override int MapOutstandingCount(DbDataReader dr) - { - int outstandingMessages = -1; - if (dr.Read()) - { - outstandingMessages = dr.GetInt32(0); - } - - dr.Close(); - - return outstandingMessages; - } - - private Message MapAMessage(DbDataReader dr) - { - var id = GetMessageId(dr); - var messageType = GetMessageType(dr); - var topic = GetTopic(dr); - - DateTimeOffset timeStamp = GetTimeStamp(dr); - var correlationId = GetCorrelationId(dr); - var replyTo = GetReplyTo(dr); - var contentType = GetContentType(dr); - var partitionKey = GetPartitionKey(dr); - - var source = GetSource(dr); - var type = GetEventType(dr); - var dataSchema = GetDataSchema(dr); - var subject = GetSubject(dr); - var specVersion = GetSpecVersion(dr); - var traceParent = GetTraceParent(dr); - var traceState = GetTraceState(dr); - var baggage = GetBaggage(dr); - var dataRef = GetDataRef(dr); - - var header = new MessageHeader( - messageId: new Id(id), - topic: topic, - messageType: messageType, - source: source, - type: type, - timeStamp: timeStamp, - correlationId: correlationId, - replyTo: replyTo , - contentType: contentType, - partitionKey: partitionKey, - dataSchema: dataSchema, - subject: subject, - handledCount: 0, // HandledCount is zero when restored from the Outbox - delayed: TimeSpan.Zero, // Delayed is zero when restored from the Outbox - traceParent: traceParent, - traceState: traceState, - baggage: baggage - ); - header.SpecVersion = specVersion ?? MessageHeader.DefaultSpecVersion; - header.DataRef = dataRef; - Dictionary? dictionaryBag = GetContextBag(dr); - if (dictionaryBag != null) - { - foreach (var keyValue in dictionaryBag) - { - header.Bag.Add(keyValue.Key, keyValue.Value); - } - } - - var body = _configuration.BinaryMessagePayload - ? new MessageBody(((NpgsqlDataReader)dr).GetFieldValue(dr.GetOrdinal("Body"))) - : new MessageBody(dr.GetString(dr.GetOrdinal("Body"))); - - return new Message(header, body); - } - - private static Baggage GetBaggage (DbDataReader dr) - { - var baggage = new Baggage(); - var (ordinal, err) = TryGetOrdinal(dr, "Baggage"); - if (err || dr.IsDBNull(ordinal)) return baggage; - - var baggageString = dr.GetString(ordinal); - if (string.IsNullOrEmpty(baggageString)) - return baggage; - - baggage.LoadBaggage(baggageString); - return baggage; - } - - private static ContentType GetContentType(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "ContentType"); - if (err || dr.IsDBNull(ordinal)) return new ContentType(MediaTypeNames.Text.Plain); - - var replyTo = dr.GetString(ordinal); - if (string.IsNullOrEmpty(replyTo)) - return new ContentType(MediaTypeNames.Text.Plain); - - return new ContentType(replyTo); - } - - private static Dictionary? GetContextBag(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "HeaderBag"); - if (err || dr.IsDBNull(ordinal)) return new Dictionary(); - - var headerBag = dr.GetString(ordinal); - var dictionaryBag = JsonSerializer.Deserialize>(headerBag, JsonSerialisationOptions.Options); - return dictionaryBag; - } - - private static Id GetCorrelationId(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "CorrelationId"); - if (err || dr.IsDBNull(ordinal)) return Id.Empty; - - var correlationId = dr.GetString(ordinal); - if (string.IsNullOrEmpty(correlationId)) - return Id.Empty; - - return new Id(correlationId); - } - - private static string GetDataRef(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "DataRef"); - if (err || dr.IsDBNull(ordinal)) return string.Empty; - - var dataRef = dr.GetString(ordinal); - return string.IsNullOrEmpty(dataRef) ? string.Empty : dataRef; - } - - private static Uri? GetDataSchema(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "DataSchema"); - if (err || dr.IsDBNull(ordinal)) return null; - - var dataSchema = dr.GetString(ordinal); - return string.IsNullOrEmpty(dataSchema) ? null : new Uri(dataSchema); - } - - private static RoutingKey GetEventType(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Type"); - if (err || dr.IsDBNull(ordinal)) return RoutingKey.Empty; - - var type = dr.GetString(ordinal); - if (string.IsNullOrEmpty(type)) - return RoutingKey.Empty; - - return new RoutingKey(type); - } - - private static RoutingKey GetTopic(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Topic"); - if (err || dr.IsDBNull(ordinal)) return RoutingKey.Empty; - - var topic = dr.GetString(ordinal); - if (string.IsNullOrEmpty(topic)) - return RoutingKey.Empty; - - return new RoutingKey(topic); - } - - private static MessageType GetMessageType(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "MessageType"); - if (err || dr.IsDBNull(ordinal)) return MessageType.MT_NONE; - - var value = dr.GetString(ordinal); - if (string.IsNullOrEmpty(value)) - return MessageType.MT_NONE; - - return (MessageType)Enum.Parse(typeof(MessageType), value); - } - - private static Id GetMessageId(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "MessageId"); - if (err || dr.IsDBNull(ordinal)) return Id.Random; - - var id = dr.GetString(ordinal); - return new Id(id); - } - - private static PartitionKey GetPartitionKey(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "PartitionKey"); - if (err || dr.IsDBNull(ordinal)) return PartitionKey.Empty; - - - var partitionKey = dr.GetString(ordinal); - return new PartitionKey(partitionKey); - } + } - private static RoutingKey GetReplyTo(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "ReplyTo"); - if (err || dr.IsDBNull(ordinal)) return RoutingKey.Empty; + /// + /// Initializes a new instance of the class. + /// + /// The configuration to connect to this data store + /// From v7.0 Npgsql uses an Npgsql data source, leave null to have Brighter manage + /// connections; Brighter will not manage type mapping for you in this case so you must register them + /// globally + public PostgreSqlOutbox( + IAmARelationalDatabaseConfiguration configuration, + NpgsqlDataSource? dataSource = null) + : this(configuration, new PostgreSqlConnectionProvider(configuration, dataSource)) + { + } - var replyTo = dr.GetString(ordinal); - if (string.IsNullOrEmpty(replyTo)) - return RoutingKey.Empty; - - return new RoutingKey(replyTo); - } - - private static Uri? GetSource(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Source"); - if (err || dr.IsDBNull(ordinal)) return null; - - var source = dr.GetString(ordinal); - return string.IsNullOrEmpty(source) ? null : new Uri(source); - } - - private static string GetSpecVersion(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "SpecVersion"); - if (err || dr.IsDBNull(ordinal)) return MessageHeader.DefaultSpecVersion; - - var specVersion = dr.GetString(ordinal); - return string.IsNullOrEmpty(specVersion) ? MessageHeader.DefaultSpecVersion : specVersion; - } - - private static string GetSubject(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Subject"); - if (err || dr.IsDBNull(ordinal)) return string.Empty; - - var subject = dr.GetString(ordinal); - return string.IsNullOrEmpty(subject) ? string.Empty : subject; - } + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) + { + return ex is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation }; + } - private static DateTimeOffset GetTimeStamp(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "Timestamp"); - if (err || dr.IsDBNull(ordinal)) return DateTimeOffset.UtcNow; - - var timeStamp = dr.IsDBNull(ordinal) - ? DateTimeOffset.MinValue - : dr.GetDateTime(ordinal); - return timeStamp; - } - - private static string GetTraceParent(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "TraceParent"); - if (err || dr.IsDBNull(ordinal)) return string.Empty; - - var traceParent = dr.GetString(ordinal); - return string.IsNullOrEmpty(traceParent) ? string.Empty : traceParent; - } - - private static string GetTraceState(DbDataReader dr) - { - var (ordinal, err) = TryGetOrdinal(dr, "TraceState"); - if (err || dr.IsDBNull(ordinal)) return string.Empty; - - var traceState = dr.GetString(ordinal); - return string.IsNullOrEmpty(traceState) ? string.Empty : traceState; - } - - private static (int, bool) TryGetOrdinal(DbDataReader dr, string columnName) - { - try - { - return (dr.GetOrdinal(columnName), false); - } - catch (IndexOutOfRangeException) - { - // SpecVersion column does not exist, return -1 and true to indicate error - return (-1, true); - } - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) + { + return new NpgsqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value }; + } - private static partial class Log - { - [LoggerMessage(LogLevel.Warning, "PostgresSqlOutbox: A duplicate was detected in the batch")] - public static partial void DuplicateDetectedInBatch(ILogger logger); - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, DbType dbType, object? value) + { + return new NpgsqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value, DbType = dbType }; } } diff --git a/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs b/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs index 503d04f056..dff5f5cd52 100644 --- a/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs +++ b/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs @@ -24,662 +24,73 @@ THE SOFTWARE. */ #endregion using System; -using System.Collections.Generic; using System.Data; using System.Data.Common; using System.Globalization; -using System.Net.Mime; -using System.IO; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Data.Sqlite; -using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Logging; using Paramore.Brighter.Observability; using Paramore.Brighter.Sqlite; -namespace Paramore.Brighter.Outbox.Sqlite +namespace Paramore.Brighter.Outbox.Sqlite; + +/// +/// Implements an outbox using Sqlite as a backing store +/// +public class SqliteOutbox : RelationDatabaseOutbox { + private const int SqliteDuplicateKeyError = 1555; + private const int SqliteUniqueKeyError = 19; + /// - /// Implements an outbox using Sqlite as a backing store + /// Initializes a new instance of the class. /// - public class SqliteOutbox : RelationDatabaseOutbox + /// The configuration to connect to this data store + /// Provides a connection to the Db that allows us to enlist in an ambient transaction + public SqliteOutbox(IAmARelationalDatabaseConfiguration configuration, + IAmARelationalDbConnectionProvider connectionProvider) + : base(DbSystem.Sqlite, configuration, connectionProvider, + new SqliteQueries(), ApplicationLogging.CreateLogger()) { - private const int SqliteDuplicateKeyError = 1555; - private const int SqliteUniqueKeyError = 19; - private readonly IAmARelationalDatabaseConfiguration _configuration; - private readonly IAmARelationalDbConnectionProvider _connectionProvider; - - /// - /// Initializes a new instance of the class. - /// - /// The configuration to connect to this data store - /// Provides a connection to the Db that allows us to enlist in an ambient transaction - public SqliteOutbox(IAmARelationalDatabaseConfiguration configuration, - IAmARelationalDbConnectionProvider connectionProvider) - : base(DbSystem.Sqlite, configuration.DatabaseName, configuration.OutBoxTableName, - new SqliteQueries(), ApplicationLogging.CreateLogger()) - { - _configuration = configuration; - ContinueOnCapturedContext = false; - _connectionProvider = connectionProvider; - } - - /// - /// Initializes a new instance of the class. - /// - /// The configuration to connect to this data store - public SqliteOutbox(IAmARelationalDatabaseConfiguration configuration) - : this(configuration, new SqliteConnectionProvider(configuration)) - { - } - - protected override void WriteToStore( - IAmABoxTransactionProvider? transactionProvider, - Func commandFunc, - Action? loggingAction - ) - { - var connection = GetOpenConnection(_connectionProvider, transactionProvider); - using var command = commandFunc.Invoke(connection); - try - { - if (transactionProvider is { HasOpenTransaction: true }) - command.Transaction = transactionProvider.GetTransaction(); - command.ExecuteNonQuery(); - } - catch (SqliteException sqlException) - { - if (!IsExceptionUnqiueOrDuplicateIssue(sqlException)) throw; - loggingAction?.Invoke(); - } - finally - { - FinishWrite(connection, transactionProvider); - } - } - - protected override async Task WriteToStoreAsync( - IAmABoxTransactionProvider? transactionProvider, - Func commandFunc, - Action? loggingAction, - CancellationToken cancellationToken) - { - var connection = await GetOpenConnectionAsync(_connectionProvider, transactionProvider, cancellationToken); - -#if NETSTANDARD - using var command = commandFunc.Invoke(connection); -#else - await using var command = commandFunc.Invoke(connection); -#endif - try - { - if (transactionProvider != null && transactionProvider.HasOpenTransaction) - command.Transaction = await transactionProvider.GetTransactionAsync(cancellationToken); - await command.ExecuteNonQueryAsync(cancellationToken); - } - catch (SqliteException sqlException) - { - if (!IsExceptionUnqiueOrDuplicateIssue(sqlException)) throw; - loggingAction?.Invoke(); - } - finally - { - if (transactionProvider != null) - transactionProvider.Close(); - else -#if NETSTANDARD2_0 - connection.Close(); -#else - await connection.CloseAsync(); -#endif - } - } - - protected override T ReadFromStore( - Func commandFunc, - Func resultFunc - ) - { - var connection = _connectionProvider.GetConnection(); - - if (connection.State != ConnectionState.Open) - connection.Open(); - using var command = commandFunc.Invoke(connection); - try - { - return resultFunc.Invoke(command.ExecuteReader()); - } - finally - { - connection.Close(); - } - } - - protected override async Task ReadFromStoreAsync( - Func commandFunc, - Func> resultFunc, - CancellationToken cancellationToken) - { - var connection = await _connectionProvider.GetConnectionAsync(cancellationToken); - - if (connection.State != ConnectionState.Open) - await connection.OpenAsync(cancellationToken); -#if NETSTANDARD2_0 - using var command = commandFunc.Invoke(connection); -#else - await using var command = commandFunc.Invoke(connection); -#endif - try - { - return await resultFunc.Invoke(await command.ExecuteReaderAsync(cancellationToken)); - } - finally - { -#if NETSTANDARD2_0 - connection.Close(); -#else - await connection.CloseAsync(); -#endif - } - } - - protected override DbCommand CreateCommand( - DbConnection connection, - string sqlText, - int outBoxTimeout, - params IDbDataParameter[] parameters) - { - var command = connection.CreateCommand(); - - command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; - command.CommandText = sqlText; - command.Parameters.AddRange(parameters); - - return command; - } - - protected override IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan since, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("TimestampSince", DateTimeOffset.UtcNow.Subtract(since)); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - - return parameters; - } - - protected override IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan dispatchedSince, int pageSize, - int pageNumber) - { - var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - - return parameters; - } - - protected override IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber) - { - var parameters = new IDbDataParameter[2]; - parameters[0] = CreateSqlParameter("Take", pageSize); - parameters[1] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - - return parameters; - } - - protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) - { - return new SqliteParameter(parameterName, value ?? DBNull.Value); - } - - protected override IDbDataParameter[] InitAddDbParameters(Message message, int? position = null) - { - var prefix = position.HasValue ? $"p{position}_" : ""; - var bagJson = JsonSerializer.Serialize(message.Header.Bag, JsonSerialisationOptions.Options); - return new IDbDataParameter[] - { - new SqliteParameter - { - ParameterName = $"@{prefix}MessageId", SqliteType = SqliteType.Text, Value = message.Id.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}MessageType", - SqliteType = SqliteType.Text, - Value = message.Header.MessageType.ToString() - }, - new SqliteParameter - { - ParameterName = $"@{prefix}Topic", - SqliteType = SqliteType.Text, - Value = message.Header.Topic.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}Timestamp", - SqliteType = SqliteType.Text, - Value = message.Header.TimeStamp.ToString("yyyy-MM-dd HH:mm:ss.fff", CultureInfo.InvariantCulture) - }, - new SqliteParameter - { - ParameterName = $"@{prefix}CorrelationId", - SqliteType = SqliteType.Text, - Value = message.Header.CorrelationId.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}ReplyTo", - SqliteType = SqliteType.Text, - Value = message.Header.ReplyTo is not null ? message.Header.ReplyTo.Value : RoutingKey.Empty.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}ContentType", - SqliteType = SqliteType.Text, - Value = message.Header.ContentType != null ? message.Header.ContentType.ToString() : new ContentType(MediaTypeNames.Text.Plain).ToString() - }, - new SqliteParameter - { - ParameterName = $"@{prefix}PartitionKey", - SqliteType = SqliteType.Text, - Value = message.Header.PartitionKey.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}HeaderBag", SqliteType = SqliteType.Text, Value = bagJson - }, - _configuration.BinaryMessagePayload - ? new SqliteParameter - { - ParameterName = $"@{prefix}Body", SqliteType = SqliteType.Blob, Value = message.Body.Bytes - } - : new SqliteParameter - { - ParameterName = $"@{prefix}Body", SqliteType = SqliteType.Text, Value = message.Body.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}Source", - SqliteType = SqliteType.Text, - Value = message.Header.Source.AbsoluteUri - }, - new SqliteParameter - { - ParameterName = $"@{prefix}Type", - SqliteType = SqliteType.Text, - Value = message.Header.Type - }, - new SqliteParameter - { - ParameterName = $"@{prefix}DataSchema", - SqliteType = SqliteType.Text, - Value = message.Header.DataSchema is not null ? message.Header.DataSchema.AbsoluteUri : "http://goparamore.io" - }, - new SqliteParameter - { - ParameterName = $"@{prefix}Subject", - SqliteType = SqliteType.Text, - Value = message.Header.Subject ?? string.Empty - }, - new SqliteParameter - { - ParameterName = $"@{prefix}TraceParent", - SqliteType = SqliteType.Text, - Value = message.Header.TraceParent is not null ? message.Header.TraceParent.Value : DBNull.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}TraceState", - SqliteType = SqliteType.Text, - Value = message.Header.TraceState is not null ? message.Header.TraceState.Value : DBNull.Value - }, - new SqliteParameter - { - ParameterName = $"@{prefix}Baggage", - SqliteType = SqliteType.Text, - Value = message.Header.Baggage.ToString() - } - }; - } - - protected override Message MapFunction(DbDataReader dr) - { - using (dr) - { - if (dr.Read()) - { - return MapAMessage(dr); - } - - return new Message(); - } - } - - protected override async Task MapFunctionAsync(DbDataReader dr, CancellationToken cancellationToken) - { - using (dr) - { - if (await dr.ReadAsync(cancellationToken)) - { - return MapAMessage(dr); - } - - return new Message(); - } - } - - protected override IEnumerable MapListFunction(DbDataReader dr) - { - var messages = new List(); - while (dr.Read()) - { - messages.Add(MapAMessage(dr)); - } - - dr.Close(); - - return messages; - } - - protected override async Task> MapListFunctionAsync(DbDataReader dr, - CancellationToken cancellationToken) - { - var messages = new List(); - while (await dr.ReadAsync(cancellationToken)) - { - messages.Add(MapAMessage(dr)); - } - -#if NETSTANDARD - dr.Close(); -#else - await dr.CloseAsync(); -#endif - - return messages; - } - - protected override async Task MapOutstandingCountAsync(DbDataReader dr, - CancellationToken cancellationToken) - { - int outstandingMessages = -1; - if (await dr.ReadAsync(cancellationToken)) - { - outstandingMessages = dr.GetInt32(0); - } - -#if NETSTANDARD - dr.Close(); -#else - await dr.CloseAsync(); -#endif - return outstandingMessages; - } - - protected override int MapOutstandingCount(DbDataReader dr) - { - int outstandingMessages = -1; - if ( dr.Read()) - { - outstandingMessages = dr.GetInt32(0); - } - - dr.Close(); - return outstandingMessages; - } - - private static bool IsExceptionUnqiueOrDuplicateIssue(SqliteException sqlException) - { - return sqlException.SqliteErrorCode == SqliteDuplicateKeyError || - sqlException.SqliteErrorCode == SqliteUniqueKeyError; - } - - private Message MapAMessage(IDataReader dr) - { - var id = GetMessageId(dr); - var messageType = GetMessageType(dr); - var topic = GetTopic(dr); - - var header = new MessageHeader(id, topic, messageType); - - if (dr.FieldCount > 4) - { - DateTimeOffset timeStamp = GetTimeStamp(dr); - var correlationId = GetCorrelationId(dr); - var replyTo = GetReplyTo(dr); - var contentType = GetContentType(dr); - var partitionKey = GetPartitionKey(dr); - var source = GetSource(dr); - var type = GetType(dr); - var dataSchema = GetDataSchema(dr); - var subject = GetSubject(dr); - var traceParent = GetTraceParent(dr); - var traceState = GetTraceState(dr); - var baggage = GetBaggage(dr); - - header = new MessageHeader( - messageId: id, - topic: topic, - messageType: messageType, - timeStamp: timeStamp, - handledCount: 0, - delayed: TimeSpan.Zero, - correlationId: correlationId is not null ? new Id(correlationId) : Id.Empty, - replyTo: replyTo is not null ? new RoutingKey(replyTo) : RoutingKey.Empty, - contentType: contentType, - partitionKey: partitionKey is not null ? new PartitionKey(partitionKey) : PartitionKey.Empty, - source: source, - type: type, - dataSchema: dataSchema, - subject: subject, - traceParent: traceParent, - traceState: traceState, - baggage: baggage - ); - - Dictionary? dictionaryBag = GetContextBag(dr); - if (dictionaryBag != null) - { - foreach (var keyValue in dictionaryBag) - { - header.Bag.Add(keyValue.Key, keyValue.Value); - } - } - } - - var body = _configuration.BinaryMessagePayload - ? new MessageBody(GetBodyAsBytes((SqliteDataReader)dr), new ContentType(MediaTypeNames.Application.Octet), CharacterEncoding.Raw) - : new MessageBody(dr.GetString(dr.GetOrdinal("Body"))); - - - return new Message(header, body); - } - - private static Baggage GetBaggage(IDataReader dr) - { - var ordinal = dr.GetOrdinal("Baggage"); - if (ordinal < 0 || ordinal >= dr.FieldCount) - return Baggage.Empty; - - return dr.IsDBNull(ordinal) - ? Baggage.Empty - : Baggage.FromString(dr.GetString(ordinal)); - } - - private static byte[] GetBodyAsBytes(SqliteDataReader dr) - { - var i = dr.GetOrdinal("Body"); - var body = dr.GetStream(i); - - if (body is MemoryStream memoryStream) // No need to dispose a MemoryStream, I do not think they dare to ever change that - return memoryStream.ToArray(); // Then we can just return its value, instead of copying manually - - MemoryStream ms = new(); - body.CopyTo(ms); - body.Dispose(); - return ms.ToArray(); - } - - private static Dictionary? GetContextBag(IDataReader dr) - { - var i = dr.GetOrdinal("HeaderBag"); - var headerBag = dr.IsDBNull(i) ? "" : dr.GetString(i); - var dictionaryBag = - JsonSerializer.Deserialize>(headerBag, JsonSerialisationOptions.Options); - return dictionaryBag; - } - - private ContentType? GetContentType(IDataReader dr) - { - var ordinal = dr.GetOrdinal("ContentType"); - if (dr.IsDBNull(ordinal)) return null; - - var contentType = dr.GetString(ordinal); - return new ContentType(contentType); - } - - private static string? GetCorrelationId(IDataReader dr) - { - var ordinal = dr.GetOrdinal("CorrelationId"); - if (dr.IsDBNull(ordinal)) return null; - - var correlationId = dr.GetString(ordinal); - return correlationId; - } - - private static Uri GetDataSchema(IDataReader dr) - { - var ordinal = dr.GetOrdinal("DataSchema"); - if (dr.IsDBNull(ordinal)) return new Uri("http://goparamore.io"); - - var uriString = dr.GetString(ordinal); - if (string.IsNullOrEmpty(uriString)) - return new Uri("http://goparamore.io"); - - return new Uri(uriString); - } - - private static MessageType GetMessageType(IDataReader dr) - { - var ordinal = dr.GetOrdinal("MessageType"); - if (dr.IsDBNull(ordinal)) return MessageType.MT_NONE; - - - var value = dr.GetString(ordinal); - if (string.IsNullOrEmpty(value)) - return MessageType.MT_NONE; - - return (MessageType)Enum.Parse(typeof(MessageType), value); - } - - private static Id GetMessageId(IDataReader dr) - { - var ordinal = dr.GetOrdinal("MessageId"); - if (dr.IsDBNull(ordinal)) return Id.Empty; - - var id = dr.GetString(ordinal); - if (string.IsNullOrEmpty(id)) - return Id.Empty; - return new Id(id); - } - - private static string? GetPartitionKey(IDataReader dr) - { - var ordinal = dr.GetOrdinal("PartitionKey"); - if (dr.IsDBNull(ordinal)) return null; - - var partitionKey = dr.GetString(ordinal); - return partitionKey; - } - - - private static string? GetReplyTo(IDataReader dr) - { - var ordinal = dr.GetOrdinal("ReplyTo"); - if (dr.IsDBNull(ordinal)) return null; - - var replyTo = dr.GetString(ordinal); - return replyTo; - } - - private static Uri GetSource(IDataReader dr) - { - var ordinal = dr.GetOrdinal("Source"); - if (dr.IsDBNull(ordinal)) return new Uri("http://goparamore.io"); - - var uriString = dr.GetString(ordinal); - if (string.IsNullOrEmpty(uriString)) - return new Uri("http://goparamore.io"); - - return new Uri(uriString); - } - - private static string GetSubject(IDataReader dr) - { - var ordinal = dr.GetOrdinal("Subject"); - if (dr.IsDBNull(ordinal)) return string.Empty; - - return dr.GetString(ordinal); - } - - private static RoutingKey GetTopic(IDataReader dr) - { - var ordinal = dr.GetOrdinal("Topic"); - if (dr.IsDBNull(ordinal)) return RoutingKey.Empty; + } + /// + /// Initializes a new instance of the class. + /// + /// The configuration to connect to this data store + public SqliteOutbox(IAmARelationalDatabaseConfiguration configuration) + : this(configuration, new SqliteConnectionProvider(configuration)) + { + } - var routingKey = dr.GetString(ordinal); - if (string.IsNullOrEmpty(routingKey)) - return RoutingKey.Empty; - - return new RoutingKey(routingKey); - } + /// + protected override bool IsExceptionUniqueOrDuplicateIssue(Exception ex) + { + return ex is SqliteException { SqliteErrorCode: SqliteDuplicateKeyError or SqliteUniqueKeyError }; + } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, object? value) + { + return new SqliteParameter(parameterName, value ?? DBNull.Value); + } - private static DateTimeOffset GetTimeStamp(IDataReader dr) - { - var ordinal = dr.GetOrdinal("Timestamp"); - var timeStamp = dr.IsDBNull(ordinal) - ? DateTimeOffset.MinValue - : dr.GetDateTime(ordinal); - return timeStamp; - } - - private static TraceParent? GetTraceParent(IDataReader dr) - { - var ordinal = dr.GetOrdinal("TraceParent"); - if (dr.IsDBNull(ordinal)) return null; - - return dr.IsDBNull(ordinal) - ? null - : new TraceParent(dr.GetString(ordinal)); - } + /// + protected override IDbDataParameter CreateSqlParameter(string parameterName, DbType dbType, object? value) + { + return new SqliteParameter(parameterName, value ?? DBNull.Value) { DbType = dbType }; + } - private static TraceState? GetTraceState(IDataReader dr) - { - var ordinal = dr.GetOrdinal("TraceState"); - - return dr.IsDBNull(ordinal) - ? null - : new TraceState(dr.GetString(ordinal)); - } - - private static string GetType(IDataReader dr) + /// + protected override DateTimeOffset GetTimeStamp(DbDataReader dr) + { + if (!TryGetOrdinal(dr, TimestampColumnName, out var ordinal) || dr.IsDBNull(ordinal)) { - var ordinal = dr.GetOrdinal("Type"); - if (dr.IsDBNull(ordinal)) return string.Empty; - - - var type = dr.GetString(ordinal); - if (string.IsNullOrEmpty(type)) - return string.Empty; - return type; + return DateTimeOffset.MinValue; } - + var reader = (SqliteDataReader)dr; + var dataTime = reader.GetDateTimeOffset(ordinal); + return dataTime; } } diff --git a/src/Paramore.Brighter/RelationDatabaseOutbox.cs b/src/Paramore.Brighter/RelationDatabaseOutbox.cs index a0ef5bc9ec..aad9283506 100644 --- a/src/Paramore.Brighter/RelationDatabaseOutbox.cs +++ b/src/Paramore.Brighter/RelationDatabaseOutbox.cs @@ -2,23 +2,32 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; +using System.IO; using System.Linq; +using System.Net.Mime; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Observability; namespace Paramore.Brighter { - public abstract class RelationDatabaseOutbox( + public abstract partial class RelationDatabaseOutbox( DbSystem dbSystem, - string databaseName, - string outboxTableName, + IAmARelationalDatabaseConfiguration configuration, + IAmARelationalDbConnectionProvider connectionProvider, IRelationDatabaseOutboxQueries queries, ILogger logger, InstrumentationOptions instrumentationOptions = InstrumentationOptions.All) : IAmAnOutboxSync, IAmAnOutboxAsync { + + protected IAmARelationalDatabaseConfiguration DatabaseConfiguration { get; } = configuration; + + protected IAmARelationalDbConnectionProvider ConnectionProvider { get; } = connectionProvider; + /// /// If false we the default thread synchronization context to run any continuation, if true we re-use the original /// synchronization context. @@ -49,14 +58,16 @@ public void Add( int outBoxTimeout = -1, IAmABoxTransactionProvider? transactionProvider = null) { - var dbAttributes = new Dictionary() + var dbAttributes = new Dictionary { - {"db.operation.parameter.message.id", message.Id.Value}, - {"db.operation.name", ExtractSqlOperationName(queries.AddCommand)}, - {"db.query.text", queries.AddCommand} + { "db.operation.parameter.message.id", message.Id.Value }, + { "db.operation.name", ExtractSqlOperationName(queries.AddCommand) }, + { "db.query.text", queries.AddCommand } }; + var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Add, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Add, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -91,22 +102,24 @@ public void Add( IAmABoxTransactionProvider? transactionProvider = null ) { - var dbAttributes = new Dictionary() + var dbAttributes = new Dictionary { - {"db.operation.parameter.message.ids", string.Join(",", messages.Select(m => m.Id))}, - {"db.operation.name", ExtractSqlOperationName(queries.BulkAddCommand)}, - {"db.query.text", queries.BulkAddCommand} + { "db.operation.parameter.message.ids", string.Join(",", messages.Select(m => m.Id)) }, + { "db.operation.name", ExtractSqlOperationName(queries.BulkAddCommand) }, + { "db.query.text", queries.BulkAddCommand } }; + var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Add, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Add, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { WriteToStore(transactionProvider, - connection => InitBulkAddDbCommand(messages.ToList(), connection), - () => logger.LogWarning("MsSqlOutbox: At least one message already exists in the outbox")); + connection => InitBulkAddDbCommand(messages.ToList(), connection), + () => logger.LogWarning("Outbox: At least one message already exists in the outbox")); } finally { @@ -130,14 +143,16 @@ public Task AddAsync( IAmABoxTransactionProvider? transactionProvider = null, CancellationToken cancellationToken = default) { - var dbAttributes = new Dictionary() + var dbAttributes = new Dictionary { - {"db.operation.parameter.message.id", message.Id.Value}, - {"db.operation.name", ExtractSqlOperationName(queries.AddCommand)}, - {"db.query.text", queries.AddCommand} + { "db.operation.parameter.message.id", message.Id.Value }, + { "db.operation.name", ExtractSqlOperationName(queries.AddCommand) }, + { "db.query.text", queries.AddCommand } }; + var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Add, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Add, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -148,7 +163,7 @@ public Task AddAsync( connection => InitAddDbCommand(connection, parameters), () => { logger.LogWarning( - "MsSqlOutbox: A duplicate Message with the MessageId {Id} was inserted into the Outbox, ignoring and continuing", + "A duplicate Message with the MessageId {Id} was inserted into the Outbox, ignoring and continuing", message.Id); }, cancellationToken); @@ -177,21 +192,22 @@ public Task AddAsync( { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.ids", string.Join(",", messages.Select(m => m.Id.Value))}, - {"db.operation.name", ExtractSqlOperationName(queries.BulkAddCommand)}, - {"db.query.text", queries.BulkAddCommand} + { "db.operation.parameter.message.ids", string.Join(",", messages.Select(m => m.Id.Value)) }, + { "db.operation.name", ExtractSqlOperationName(queries.BulkAddCommand) }, + { "db.query.text", queries.BulkAddCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Add, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Add, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { return WriteToStoreAsync(transactionProvider, - connection => InitBulkAddDbCommand(messages.ToList(), connection), - () => logger.LogWarning("MsSqlOutbox: At least one message already exists in the outbox"), - cancellationToken); + connection => InitBulkAddDbCommand(messages.ToList(), connection), + () => logger.LogWarning("MsSqlOutbox: At least one message already exists in the outbox"), + cancellationToken); } finally { @@ -209,19 +225,22 @@ public void Delete(Id[] messageIds, RequestContext? requestContext, Dictionary() { - {"db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.Value))}, - {"db.operation.name", ExtractSqlOperationName(queries.DeleteMessagesCommand)}, - {"db.query.text", queries.DeleteMessagesCommand} + { "db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.Value)) }, + { "db.operation.name", ExtractSqlOperationName(queries.DeleteMessagesCommand) }, + { "db.query.text", queries.DeleteMessagesCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Delete, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Delete, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { if (messageIds.Any()) - WriteToStore(null, connection => InitDeleteDispatchedCommand(connection, messageIds.Select(m => m.ToString())), null); + WriteToStore(null, + connection => InitDeleteDispatchedCommand(connection, messageIds.Select(m => m.ToString())), + null); } finally { @@ -244,13 +263,14 @@ public Task DeleteAsync( { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.Value))}, - {"db.operation.name", ExtractSqlOperationName(queries.DeleteMessagesCommand)}, - {"db.query.text", queries.DeleteMessagesCommand}, + { "db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.Value)) }, + { "db.operation.name", ExtractSqlOperationName(queries.DeleteMessagesCommand) }, + { "db.query.text", queries.DeleteMessagesCommand }, }; - + var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Delete, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Delete, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -259,7 +279,8 @@ public Task DeleteAsync( if (!messageIds.Any()) return Task.CompletedTask; - return WriteToStoreAsync(null, connection => InitDeleteDispatchedCommand(connection, messageIds.Select(m => m.ToString())), null, + return WriteToStoreAsync(null, + connection => InitDeleteDispatchedCommand(connection, messageIds.Select(m => m.ToString())), null, cancellationToken); } finally @@ -290,20 +311,21 @@ public async Task> DispatchedMessagesAsync( { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.PagedDispatchedCommand)}, - {"db.query.text", queries.PagedDispatchedCommand} + { "db.operation.name", ExtractSqlOperationName(queries.PagedDispatchedCommand) }, + { "db.query.text", queries.PagedDispatchedCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.DispatchedMessages, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.DispatchedMessages, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { var result = await ReadFromStoreAsync( - connection => - CreatePagedDispatchedCommand(connection, dispatchedSince, pageSize, pageNumber, outboxTimeout), - dr => MapListFunctionAsync(dr, cancellationToken), cancellationToken); + connection => + CreatePagedDispatchedCommand(connection, dispatchedSince, pageSize, pageNumber, outboxTimeout), + dr => MapListFunctionAsync(dr, cancellationToken), cancellationToken); span?.AddTag("db.response.returned_rows", result.Count()); return result; @@ -356,20 +378,21 @@ public IEnumerable DispatchedMessages( { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.PagedDispatchedCommand)}, - {"db.query.text", queries.PagedDispatchedCommand} + { "db.operation.name", ExtractSqlOperationName(queries.PagedDispatchedCommand) }, + { "db.query.text", queries.PagedDispatchedCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.DispatchedMessages, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.DispatchedMessages, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { var result = ReadFromStore( - connection => - CreatePagedDispatchedCommand(connection, dispatchedSince, pageSize, pageNumber, outBoxTimeout), - MapListFunction); + connection => + CreatePagedDispatchedCommand(connection, dispatchedSince, pageSize, pageNumber, outBoxTimeout), + MapListFunction); span?.AddTag("db.response.returned_rows", result.Count()); return result; @@ -400,21 +423,22 @@ public IEnumerable DispatchedMessages( { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.PagedDispatchedCommand)}, - {"db.query.text", queries.PagedDispatchedCommand} + { "db.operation.name", ExtractSqlOperationName(queries.PagedDispatchedCommand) }, + { "db.query.text", queries.PagedDispatchedCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.DispatchedMessages, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.DispatchedMessages, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { var result = ReadFromStore( - connection => - CreatePagedDispatchedCommand(connection, TimeSpan.FromHours(hoursDispatchedSince), pageSize, - pageNumber, outBoxTimeout), - MapListFunction); + connection => + CreatePagedDispatchedCommand(connection, TimeSpan.FromHours(hoursDispatchedSince), pageSize, + pageNumber, outBoxTimeout), + MapListFunction); span?.AddTag("db.response.returned_rows", result.Count()); return result; @@ -434,25 +458,28 @@ public IEnumerable DispatchedMessages( /// For outboxes that require additional parameters such as topic, provide an optional arg /// The message public IEnumerable Get( - IEnumerable messageIds, + IEnumerable messageIds, RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null) { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.ids", string.Join(",", messageIds)}, - {"db.operation.name", ExtractSqlOperationName(queries.GetMessagesCommand)}, - {"db.query.text", queries.GetMessagesCommand} + { "db.operation.parameter.message.ids", string.Join(",", messageIds) }, + { "db.operation.name", ExtractSqlOperationName(queries.GetMessagesCommand) }, + { "db.query.text", queries.GetMessagesCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { - var result = ReadFromStore(connection => InitGetMessagesCommand(connection, messageIds.Select(m => m.ToString()).ToList(), outBoxTimeout), + var result = ReadFromStore( + connection => InitGetMessagesCommand(connection, messageIds.Select(m => m.ToString()).ToList(), + outBoxTimeout), MapListFunction); span?.AddTag("db.response.returned_rows", result.Count()); @@ -473,27 +500,28 @@ public IEnumerable Get( /// For outboxes that require additional parameters such as topic, provide an optional arg /// The message public Message Get( - Id messageId, - RequestContext requestContext, + Id messageId, + RequestContext requestContext, int outBoxTimeout = -1, Dictionary? args = null - ) + ) { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.id", messageId.Value}, - {"db.operation.name", ExtractSqlOperationName(queries.GetMessageCommand)}, - {"db.query.text", queries.GetMessageCommand} + { "db.operation.parameter.message.id", messageId.Value }, + { "db.operation.name", ExtractSqlOperationName(queries.GetMessageCommand) }, + { "db.query.text", queries.GetMessageCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { var message = ReadFromStore(connection => InitGetMessageCommand(connection, messageId, outBoxTimeout), - MapFunction); + MapFunction); return message; } @@ -521,20 +549,21 @@ public async Task GetAsync( { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.id", messageId.Value}, - {"db.operation.name", ExtractSqlOperationName(queries.GetMessageCommand)}, - {"db.query.text", queries.GetMessageCommand} + { "db.operation.parameter.message.id", messageId.Value }, + { "db.operation.name", ExtractSqlOperationName(queries.GetMessageCommand) }, + { "db.query.text", queries.GetMessageCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { var message = await ReadFromStoreAsync( - connection => InitGetMessageCommand(connection, messageId, outBoxTimeout), - dr => MapFunctionAsync(dr, cancellationToken), cancellationToken); + connection => InitGetMessageCommand(connection, messageId, outBoxTimeout), + dr => MapFunctionAsync(dr, cancellationToken), cancellationToken); return message; } @@ -561,20 +590,22 @@ public async Task> GetAsync( { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.Value))}, - {"db.operation.name", ExtractSqlOperationName(queries.GetMessagesCommand)}, - {"db.query.text", queries.GetMessagesCommand} + { "db.operation.parameter.message.ids", string.Join(",", messageIds.Select(id => id.Value)) }, + { "db.operation.name", ExtractSqlOperationName(queries.GetMessagesCommand) }, + { "db.query.text", queries.GetMessagesCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { var result = await ReadFromStoreAsync( - connection => InitGetMessagesCommand(connection, messageIds.Select(m => m.Value).ToList(), outBoxTimeout), - async (dr) => await MapListFunctionAsync(dr, cancellationToken), cancellationToken); + connection => + InitGetMessagesCommand(connection, messageIds.Select(m => m.Value).ToList(), outBoxTimeout), + async (dr) => await MapListFunctionAsync(dr, cancellationToken), cancellationToken); span?.AddTag("db.response.returned_rows", result.Count()); return result; @@ -592,22 +623,24 @@ public async Task> GetAsync( /// Page number of results to return (default = 1) /// Additional parameters required for search, if any /// A list of messages - public IList Get(RequestContext? requestContext, int pageSize = 100, int pageNumber = 1, Dictionary? args = null) + public IList Get(RequestContext? requestContext, int pageSize = 100, int pageNumber = 1, + Dictionary? args = null) { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.PagedReadCommand)}, - {"db.query.text", queries.PagedReadCommand} + { "db.operation.name", ExtractSqlOperationName(queries.PagedReadCommand) }, + { "db.query.text", queries.PagedReadCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { var result = ReadFromStore(connection => CreatePagedReadCommand(connection, pageSize, pageNumber), - MapListFunction).ToList(); + MapListFunction).ToList(); span?.AddTag("db.response.returned_rows", result.Count()); return result; @@ -636,18 +669,20 @@ public async Task> GetAsync( { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.PagedReadCommand)}, - {"db.query.text", queries.PagedReadCommand} + { "db.operation.name", ExtractSqlOperationName(queries.PagedReadCommand) }, + { "db.query.text", queries.PagedReadCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { - var result = (await ReadFromStoreAsync(connection => CreatePagedReadCommand(connection, pageSize, pageNumber), - dr => MapListFunctionAsync(dr, cancellationToken), cancellationToken)).ToList(); + var result = (await ReadFromStoreAsync( + connection => CreatePagedReadCommand(connection, pageSize, pageNumber), + dr => MapListFunctionAsync(dr, cancellationToken), cancellationToken)).ToList(); span?.AddTag("db.response.returned_rows", result.Count()); return result; @@ -663,15 +698,17 @@ public async Task> GetAsync( /// /// Cancel the async operation /// - public async Task GetNumberOfOutstandingMessagesAsync(RequestContext? requestContext, CancellationToken cancellationToken = default) + public async Task GetNumberOfOutstandingMessagesAsync(RequestContext? requestContext, + CancellationToken cancellationToken = default) { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.GetNumberOfOutstandingMessagesCommand)}, - {"db.query.text", queries.GetNumberOfOutstandingMessagesCommand} + { "db.operation.name", ExtractSqlOperationName(queries.GetNumberOfOutstandingMessagesCommand) }, + { "db.query.text", queries.GetNumberOfOutstandingMessagesCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -694,11 +731,12 @@ public int GetNumberOfOutstandingMessages(RequestContext? requestContext) { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.GetNumberOfOutstandingMessagesCommand)}, - {"db.query.text", queries.GetNumberOfOutstandingMessagesCommand} + { "db.operation.name", ExtractSqlOperationName(queries.GetNumberOfOutstandingMessagesCommand) }, + { "db.query.text", queries.GetNumberOfOutstandingMessagesCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -729,20 +767,22 @@ public async Task MarkDispatchedAsync( { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.id", id.Value}, - {"db.operation.name", ExtractSqlOperationName(queries.MarkDispatchedCommand)}, - {"db.query.text", queries.MarkDispatchedCommand} + { "db.operation.parameter.message.id", id.Value }, + { "db.operation.name", ExtractSqlOperationName(queries.MarkDispatchedCommand) }, + { "db.query.text", queries.MarkDispatchedCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.MarkDispatched, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.MarkDispatched, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { await WriteToStoreAsync(null, - connection => InitMarkDispatchedCommand(connection, id, dispatchedAt ?? DateTimeOffset.UtcNow), null, - cancellationToken); + connection => InitMarkDispatchedCommand(connection, id, dispatchedAt ?? DateTimeOffset.UtcNow), + null, + cancellationToken); } finally { @@ -767,19 +807,21 @@ public async Task MarkDispatchedAsync( { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.ids", string.Join(",", ids.Select(m => m.Value))}, - {"db.operation.name", ExtractSqlOperationName(queries.MarkMultipleDispatchedCommand)}, - {"db.query.text", queries.MarkMultipleDispatchedCommand} + { "db.operation.parameter.message.ids", string.Join(",", ids.Select(m => m.Value)) }, + { "db.operation.name", ExtractSqlOperationName(queries.MarkMultipleDispatchedCommand) }, + { "db.query.text", queries.MarkMultipleDispatchedCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.MarkDispatched, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.MarkDispatched, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { await WriteToStoreAsync(null, - connection => InitMarkDispatchedCommand(connection, ids, dispatchedAt ?? DateTimeOffset.UtcNow), null, + connection => InitMarkDispatchedCommand(connection, ids, dispatchedAt ?? DateTimeOffset.UtcNow), + null, cancellationToken); } finally @@ -796,25 +838,27 @@ await WriteToStoreAsync(null, /// When was the message dispatched, defaults to UTC now /// Allows additional arguments to be provided for specific Outbox Db providers public void MarkDispatched( - Id id, - RequestContext requestContext, + Id id, + RequestContext requestContext, DateTimeOffset? dispatchedAt = null, Dictionary? args = null) { var dbAttributes = new Dictionary() { - {"db.operation.parameter.message.id", id.Value}, - {"db.operation.name", ExtractSqlOperationName(queries.MarkDispatchedCommand)}, - {"db.query.text", queries.MarkDispatchedCommand} + { "db.operation.parameter.message.id", id.Value }, + { "db.operation.name", ExtractSqlOperationName(queries.MarkDispatchedCommand) }, + { "db.query.text", queries.MarkDispatchedCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.MarkDispatched, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.MarkDispatched, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); try { - WriteToStore(null, connection => InitMarkDispatchedCommand(connection, id, dispatchedAt ?? DateTime.UtcNow), + WriteToStore(null, + connection => InitMarkDispatchedCommand(connection, id, dispatchedAt ?? DateTime.UtcNow), null); } finally @@ -841,11 +885,12 @@ public IEnumerable OutstandingMessages( { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.PagedOutstandingCommand)}, - {"db.query.text", queries.PagedOutstandingCommand} + { "db.operation.name", ExtractSqlOperationName(queries.PagedOutstandingCommand) }, + { "db.query.text", queries.PagedOutstandingCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.OutStandingMessages, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.OutStandingMessages, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -884,11 +929,12 @@ public async Task> OutstandingMessagesAsync( { var dbAttributes = new Dictionary() { - {"db.operation.name", ExtractSqlOperationName(queries.PagedOutstandingCommand)}, - {"db.query.text", queries.PagedOutstandingCommand} + { "db.operation.name", ExtractSqlOperationName(queries.PagedOutstandingCommand) }, + { "db.query.text", queries.PagedOutstandingCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.OutStandingMessages, outboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.OutStandingMessages, DatabaseConfiguration.OutBoxTableName, + dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -907,31 +953,137 @@ public async Task> OutstandingMessagesAsync( } } - protected abstract void WriteToStore( + protected virtual void WriteToStore( IAmABoxTransactionProvider? transactionProvider, Func commandFunc, Action? loggingAction - ); + ) + { + var connection = GetOpenConnection(ConnectionProvider, transactionProvider); + + using var command = commandFunc.Invoke(connection); + try + { + if (transactionProvider is { HasOpenTransaction: true }) + { + command.Transaction = transactionProvider.GetTransaction(); + } + + command.ExecuteNonQuery(); + } + catch (DbException exception) + { + if (!IsExceptionUniqueOrDuplicateIssue(exception)) + { + throw; + } - protected abstract Task WriteToStoreAsync( + loggingAction?.Invoke(); + Log.DuplicateDetectedInBatch(logger); + } + finally + { + FinishWrite(connection, transactionProvider); + } + } + + protected virtual async Task WriteToStoreAsync( IAmABoxTransactionProvider? transactionProvider, Func commandFunc, Action? loggingAction, CancellationToken cancellationToken - ); + ) + { + var connection = await GetOpenConnectionAsync(ConnectionProvider, transactionProvider, cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + +#if NETSTANDARD + using var command = commandFunc.Invoke(connection); +#else + await using var command = commandFunc.Invoke(connection); +#endif + + try + { + if (transactionProvider is { HasOpenTransaction: true }) + { + command.Transaction = await transactionProvider.GetTransactionAsync(cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + } + + await command + .ExecuteNonQueryAsync(cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + } + catch (DbException exception) + { + if (!IsExceptionUniqueOrDuplicateIssue(exception)) + { + throw; + } + + loggingAction?.Invoke(); + } + finally + { + FinishWrite(connection, transactionProvider); + } + } - protected abstract T ReadFromStore( + protected abstract bool IsExceptionUniqueOrDuplicateIssue(Exception ex); + + protected virtual T ReadFromStore( Func commandFunc, Func resultFunc - ); + ) + { + var connection = GetOpenConnection(ConnectionProvider, null); + using var command = commandFunc.Invoke(connection); + try + { + return resultFunc.Invoke(command.ExecuteReader()); + } + finally + { + connection.Close(); + } + } - protected abstract Task ReadFromStoreAsync( + protected virtual async Task ReadFromStoreAsync( Func commandFunc, Func> resultFunc, CancellationToken cancellationToken - ); + ) + { + var connection = await GetOpenConnectionAsync(ConnectionProvider, null, cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + +#if NETSTANDARD + using var command = commandFunc.Invoke(connection); +#else + await using var command = commandFunc.Invoke(connection); +#endif + try + { + var dr = await command.ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + + return await resultFunc.Invoke(dr) + .ConfigureAwait(ContinueOnCapturedContext); + } + finally + { +#if NETSTANDARD + connection.Close(); +#else + await connection + .CloseAsync() + .ConfigureAwait(ContinueOnCapturedContext); +#endif + } + } - protected DbConnection GetOpenConnection(IAmARelationalDbConnectionProvider defaultConnectionProvider, + protected virtual DbConnection GetOpenConnection(IAmARelationalDbConnectionProvider defaultConnectionProvider, IAmABoxTransactionProvider? transactionProvider) { var connectionProvider = defaultConnectionProvider; @@ -946,7 +1098,7 @@ protected DbConnection GetOpenConnection(IAmARelationalDbConnectionProvider defa return connection; } - protected void FinishWrite(DbConnection connection, + protected virtual void FinishWrite(DbConnection connection, IAmABoxTransactionProvider? transactionProvider) { if (transactionProvider != null) @@ -955,7 +1107,7 @@ protected void FinishWrite(DbConnection connection, connection.Close(); } - protected async Task GetOpenConnectionAsync( + protected virtual async Task GetOpenConnectionAsync( IAmARelationalDbConnectionProvider defaultConnectionProvider, IAmABoxTransactionProvider? transactionProvider, CancellationToken cancellationToken) { @@ -1018,7 +1170,8 @@ private DbCommand InitMarkDispatchedCommand(DbConnection connection, Id messageI CreateSqlParameter("MessageId", messageId.Value), CreateSqlParameter("DispatchedAt", dispatchedAt?.ToUniversalTime())); - private DbCommand InitMarkDispatchedCommand(DbConnection connection, IEnumerable messageIds, DateTimeOffset? dispatchedAt) + private DbCommand InitMarkDispatchedCommand(DbConnection connection, IEnumerable messageIds, + DateTimeOffset? dispatchedAt) { var inClause = GenerateInClauseAndAddParameters(messageIds.Select(m => m.ToString()).ToList()); return CreateCommand(connection, GenerateSqlText(queries.MarkMultipleDispatchedCommand, inClause.inClause), @@ -1041,7 +1194,7 @@ private DbCommand InitGetMessagesCommand(DbConnection connection, List m } private string GenerateSqlText(string sqlFormat, params string[] orderedParams) - => string.Format(sqlFormat, orderedParams.Prepend(outboxTableName).ToArray()); + => string.Format(sqlFormat, orderedParams.Prepend(DatabaseConfiguration.OutBoxTableName).ToArray()); private DbCommand InitDeleteDispatchedCommand(DbConnection connection, IEnumerable messageIds) { @@ -1050,32 +1203,159 @@ private DbCommand InitDeleteDispatchedCommand(DbConnection connection, IEnumerab inClause.parameters); } - protected abstract DbCommand CreateCommand(DbConnection connection, string sqlText, int outBoxTimeout, - params IDbDataParameter[] parameters); + protected virtual DbCommand CreateCommand(DbConnection connection, string sqlText, int outBoxTimeout, + params IDbDataParameter[] parameters) + { + var command = connection.CreateCommand(); + + command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; + command.CommandText = sqlText; + command.Parameters.AddRange(parameters); + + return command; + } + + protected virtual IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan since, int pageSize, + int pageNumber) + { + var parameters = new IDbDataParameter[3]; + parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); + parameters[1] = CreateSqlParameter("Take", pageSize); + parameters[2] = CreateSqlParameter("TimestampSince", DateTimeOffset.UtcNow.Subtract(since)); - protected abstract IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan since, int pageSize, - int pageNumber); + return parameters; + } - protected abstract IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan dispatchedSince, int pageSize, - int pageNumber); + protected virtual IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan dispatchedSince, int pageSize, int pageNumber) + { + + var parameters = new IDbDataParameter[3]; + parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); + parameters[1] = CreateSqlParameter("Take", pageSize); + parameters[2] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); - protected abstract IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber); + return parameters; + } + + protected virtual IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber) + { + var parameters = new IDbDataParameter[2]; + parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); + parameters[1] = CreateSqlParameter("Take", pageSize); + + return parameters; + } protected abstract IDbDataParameter CreateSqlParameter(string parameterName, object? value); - protected abstract IDbDataParameter[] InitAddDbParameters(Message message, int? position = null); + protected abstract IDbDataParameter CreateSqlParameter(string parameterName, DbType dbType, object? value); - protected abstract Message MapFunction(DbDataReader dr); + protected virtual IDbDataParameter[] InitAddDbParameters(Message message, int? position = null) + { + var prefix = position.HasValue ? $"p{position}_" : ""; + var bagJson = JsonSerializer.Serialize(message.Header.Bag, JsonSerialisationOptions.Options); + + var body = DatabaseConfiguration.BinaryMessagePayload ? + CreateSqlParameter($"@{prefix}Body", DbType.Byte, message.Body.Bytes) + : CreateSqlParameter($"@{prefix}Body", DbType.String, message.Body.Value); + + return + [ + body, + CreateSqlParameter($"@{prefix}MessageId", DbType.String, message.Id.Value), + CreateSqlParameter($"@{prefix}MessageType", DbType.String, message.Header.MessageType.ToString()), + CreateSqlParameter($"@{prefix}Topic", DbType.String, message.Header.Topic.Value), + CreateSqlParameter($"@{prefix}Timestamp", DbType.DateTimeOffset, message.Header.TimeStamp.ToUniversalTime()), + CreateSqlParameter($"@{prefix}CorrelationId", DbType.String, message.Header.CorrelationId.Value), + CreateSqlParameter($"@{prefix}ReplyTo", DbType.String, message.Header.ReplyTo?.Value), + CreateSqlParameter($"@{prefix}ContentType", DbType.String, message.Header.ContentType.ToString()), + CreateSqlParameter($"@{prefix}PartitionKey", DbType.String, message.Header.PartitionKey.Value), + CreateSqlParameter($"@{prefix}HeaderBag", DbType.String, bagJson), + CreateSqlParameter($"@{prefix}Source", DbType.String, message.Header.Source.ToString()), + CreateSqlParameter($"@{prefix}Type", DbType.String, message.Header.Type), + CreateSqlParameter($"@{prefix}DataSchema", DbType.String, message.Header.DataSchema?.ToString()), + CreateSqlParameter($"@{prefix}Subject", DbType.String, message.Header.Subject), + CreateSqlParameter($"@{prefix}TraceParent", DbType.String, message.Header.TraceParent?.Value), + CreateSqlParameter($"@{prefix}TraceState", DbType.String, message.Header.TraceState?.Value), + CreateSqlParameter($"@{prefix}Baggage", DbType.String, message.Header.Baggage.ToString()), + ]; + } - protected abstract Task MapFunctionAsync(DbDataReader dr, CancellationToken cancellationToken); + protected virtual Message MapFunction(DbDataReader dr) + { + return dr.Read() ? MapAMessage(dr) : new Message(); + } - protected abstract IEnumerable MapListFunction(DbDataReader dr); + protected virtual async Task MapFunctionAsync(DbDataReader dr, CancellationToken cancellationToken) + { + if (await dr.ReadAsync(cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext)) + { + return MapAMessage(dr); + } - protected abstract Task> MapListFunctionAsync(DbDataReader dr, - CancellationToken cancellationToken); + return new Message();} - protected abstract Task MapOutstandingCountAsync(DbDataReader dr, CancellationToken cancellationToken); - protected abstract int MapOutstandingCount(DbDataReader dr); + protected virtual IEnumerable MapListFunction(DbDataReader dr) + { + var messages = new List(); + while (dr.Read()) + { + messages.Add(MapAMessage(dr)); + } + + dr.Close(); + + return messages; + } + + protected virtual async Task> MapListFunctionAsync(DbDataReader dr, + CancellationToken cancellationToken) + { + + var messages = new List(); + while (await dr.ReadAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext)) + { + messages.Add(MapAMessage(dr)); + } + +#if NETSTANDARD2_0 + dr.Close(); +#else + await dr.CloseAsync().ConfigureAwait(ContinueOnCapturedContext); +#endif + + return messages; + } + + protected virtual int MapOutstandingCount(DbDataReader dr) + { + int outstandingMessages = -1; + if (dr.Read()) + { + outstandingMessages = dr.GetInt32(0); + } + + dr.Close(); + return outstandingMessages; + } + + protected virtual async Task MapOutstandingCountAsync(DbDataReader dr, CancellationToken cancellationToken) + { + int outstandingMessages = -1; + if (await dr.ReadAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext)) + { + outstandingMessages = dr.GetInt32(0); + } + +#if NETSTANDARD2_0 + dr.Close(); +#else + await dr.CloseAsync().ConfigureAwait(ContinueOnCapturedContext); +#endif + + return outstandingMessages; + } private (string inClause, IDbDataParameter[] parameters) GenerateInClauseAndAddParameters( List messageIds) @@ -1083,7 +1363,7 @@ protected abstract Task> MapListFunctionAsync(DbDataReader var paramNames = messageIds.Select((s, i) => "@p" + i).ToArray(); var parameters = new IDbDataParameter[messageIds.Count]; - for (int i = 0; i < paramNames.Count(); i++) + for (int i = 0; i < paramNames.Length; i++) { parameters[i] = CreateSqlParameter(paramNames[i], messageIds[i]); } @@ -1094,9 +1374,9 @@ protected abstract Task> MapListFunctionAsync(DbDataReader private (string insertClause, IDbDataParameter[] parameters) GenerateBulkInsert(List messages) { var messageParams = new List(); - var parameters = new List(); + var parameters = new List(); - for (int i = 0; i < messages.Count(); i++) + for (int i = 0; i < messages.Count; i++) { // include all columns in the same order as the CREATE TABLE DDL: messageParams.Add( @@ -1114,5 +1394,341 @@ private static string ExtractSqlOperationName(string queryText) { return queryText.Split(' ')[0]; } + + protected virtual Message MapAMessage(DbDataReader dr) + { + var header = new MessageHeader( + messageId: GetMessageId(dr), + topic: GetTopic(dr), + messageType: GetMessageType(dr), + source: GetSource(dr), + type: GetEventType(dr), + timeStamp: GetTimeStamp(dr), + correlationId: GetCorrelationId(dr), + replyTo: GetReplyTo(dr), + contentType: GetContentType(dr), + partitionKey: GetPartitionKey(dr), + dataSchema: GetDataSchema(dr), + subject: GetSubject(dr), + handledCount: 0, // HandledCount is zero when restored from the Outbox + delayed: TimeSpan.Zero, // Delayed is zero when restored from the Outbox + traceParent: GetTraceParent(dr), + traceState: GetTraceState(dr), + baggage: GetBaggage(dr) + ) + { + SpecVersion = GetSpecVersion(dr), + DataRef = GetDataRef(dr) + }; + + Dictionary? dictionaryBag = GetContextBag(dr); + if (dictionaryBag != null) + { + foreach (var keyValue in dictionaryBag) + { + header.Bag.Add(keyValue.Key, keyValue.Value); + } + } + + var body = DatabaseConfiguration.BinaryMessagePayload + ? new MessageBody(GetBodyAsByteArray(dr)) + : new MessageBody(GetBodyAsString(dr)); + + return new Message(header, body); + } + + protected virtual bool TryGetOrdinal(DbDataReader dr, string columnName, out int ordinal) + { + try + { + ordinal = dr.GetOrdinal(columnName); + return true; + } + catch (ArgumentOutOfRangeException) + { + ordinal = -1; + return false; + } + catch (IndexOutOfRangeException) + { + // SpecVersion column does not exist, return -1 and true to indicate error + ordinal = -1; + return false; + } + } + + protected virtual string BodyColumnName => "Body"; + protected virtual byte[] GetBodyAsByteArray(DbDataReader dr) + { + var body = dr.GetStream(dr.GetOrdinal(BodyColumnName)); + if (body is MemoryStream memoryStream) // No need to dispose a MemoryStream, I do not think they dare to ever change that + { + return memoryStream.ToArray(); // Then we can just return its value, instead of copying manually + } + + var ms = new MemoryStream(); + body.CopyTo(ms); + body.Dispose(); + return ms.ToArray(); + } + + protected virtual string GetBodyAsString(DbDataReader dr) => dr.GetString(dr.GetOrdinal(BodyColumnName)); + + + protected virtual string BaggageColumnName => "Baggage"; + protected virtual Baggage GetBaggage(DbDataReader dr) + { + var baggage = new Baggage(); + if (!TryGetOrdinal(dr, BaggageColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return baggage; + } + + var baggageString = dr.GetString(ordinal); + if (string.IsNullOrEmpty(baggageString)) + { + return baggage; + } + + baggage.LoadBaggage(baggageString); + return baggage; + } + + protected virtual string ContentTypeColumnName => "ContentType"; + protected virtual ContentType GetContentType(DbDataReader dr) + { + if (!TryGetOrdinal(dr, ContentTypeColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return new ContentType(MediaTypeNames.Text.Plain); + } + + var replyTo = dr.GetString(ordinal); + return string.IsNullOrEmpty(replyTo) ? new ContentType(MediaTypeNames.Text.Plain) : new ContentType(replyTo); + } + + protected virtual string HeaderBagColumnName => "HeaderBag"; + protected virtual Dictionary? GetContextBag(DbDataReader dr) + { + if (!TryGetOrdinal(dr, HeaderBagColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return new Dictionary(); + } + + var headerBag = dr.GetString(ordinal); + var dictionaryBag = JsonSerializer.Deserialize>(headerBag, JsonSerialisationOptions.Options); + return dictionaryBag; + } + + protected virtual string CorrelationIdColumnName => "CorrelationId"; + protected virtual Id GetCorrelationId(DbDataReader dr) + { + if (!TryGetOrdinal(dr, CorrelationIdColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return Id.Empty; + } + + var correlationId = dr.GetString(ordinal); + if (string.IsNullOrEmpty(correlationId)) + { + return Id.Empty; + } + + return new Id(correlationId); + } + + protected virtual string DataRefColumnName => "DataRef"; + protected virtual string? GetDataRef(DbDataReader dr) + { + if (!TryGetOrdinal(dr, DataRefColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return null; + } + + return dr.GetString(ordinal); + } + + protected virtual string PartitionKeyColumnName => "PartitionKey"; + protected virtual PartitionKey GetPartitionKey(DbDataReader dr) + { + if (!TryGetOrdinal(dr, PartitionKeyColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return string.Empty; + } + + var partitionKey = dr.GetString(ordinal); + return string.IsNullOrEmpty(partitionKey) ? PartitionKey.Empty : new PartitionKey(partitionKey); + } + + protected virtual string DataSchemaColumnName => "DataSchema"; + protected virtual Uri? GetDataSchema(DbDataReader dr) + { + if (!TryGetOrdinal(dr, DataSchemaColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return null; + } + + return Uri.TryCreate(dr.GetString(ordinal), UriKind.Absolute, out var uri) ? uri : null; + } + + protected virtual string TypeColumnName => "Type"; + protected virtual string GetEventType(DbDataReader dr) + { + if (!TryGetOrdinal(dr, TypeColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return MessageHeader.DefaultType; + } + + var type = dr.GetString(ordinal); + if (string.IsNullOrEmpty(type)) + { + return MessageHeader.DefaultType; + } + + return type; + } + + protected virtual string TopicColumnName => "Topic"; + protected virtual RoutingKey GetTopic(DbDataReader dr) + { + if (!TryGetOrdinal(dr, TopicColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return RoutingKey.Empty; + } + + var topic = dr.GetString(ordinal); + if (string.IsNullOrEmpty(topic)) + { + return RoutingKey.Empty; + } + + return new RoutingKey(topic); + } + + protected virtual string ReplyToColumnName => "ReplyTo"; + protected virtual RoutingKey? GetReplyTo(DbDataReader dr) + { + if (!TryGetOrdinal(dr, ReplyToColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return null; + } + + var topic = dr.GetString(ordinal); + if (string.IsNullOrEmpty(topic)) + { + return null; + } + + return new RoutingKey(topic); + } + + protected virtual string MessageTypeColumnName => "MessageType"; + protected virtual MessageType GetMessageType(DbDataReader dr) + { + if (!TryGetOrdinal(dr, MessageTypeColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return MessageType.MT_NONE; + } + + var value = dr.GetString(ordinal); + if (string.IsNullOrEmpty(value)) + { + return MessageType.MT_NONE; + } + +#if NETSTANDARD + return (MessageType)Enum.Parse(typeof(MessageType), value); +#else + return Enum.Parse(value); +#endif + } + + protected virtual string MessageIdColumnName => "MessageId"; + protected virtual Id GetMessageId(DbDataReader dr) + { + if (!TryGetOrdinal(dr, MessageIdColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return Id.Random; + } + + var id = dr.GetString(ordinal); + return new Id(id); + } + + protected virtual string SourceColumnName => "Source"; + protected virtual Uri GetSource(DbDataReader dr) + { + if (!TryGetOrdinal(dr, SourceColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return new Uri(MessageHeader.DefaultSource); + } + + return Uri.TryCreate(dr.GetString(ordinal), UriKind.RelativeOrAbsolute, out var source) ? source : new Uri(MessageHeader.DefaultSource); + } + + protected virtual string SpecVersionColumnName => "SpecVersion"; + protected virtual string GetSpecVersion(DbDataReader dr) + { + if (!TryGetOrdinal(dr, SpecVersionColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return MessageHeader.DefaultSpecVersion; + } + + var specVersion = dr.GetString(ordinal); + return string.IsNullOrEmpty(specVersion) ? MessageHeader.DefaultSpecVersion : specVersion; + } + + protected virtual string SubjectColumnName => "Subject"; + protected virtual string? GetSubject(DbDataReader dr) + { + if (!TryGetOrdinal(dr, SubjectColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return null; + } + + var subject = dr.GetString(ordinal); + return string.IsNullOrEmpty(subject) ? null : subject; + } + + protected virtual string TimestampColumnName => "Timestamp"; + protected virtual DateTimeOffset GetTimeStamp(DbDataReader dr) + { + if (!TryGetOrdinal(dr, TimestampColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return DateTimeOffset.UtcNow; + } + + var dataTime = dr.GetDateTime(ordinal); + return dataTime; + } + + protected virtual string TraceParentColumnName => "TraceParent"; + protected virtual TraceParent? GetTraceParent(DbDataReader dr) + { + if (!TryGetOrdinal(dr, TraceParentColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return null; + } + + var traceParent = dr.GetString(ordinal); + return string.IsNullOrEmpty(traceParent) ? null : new TraceParent(traceParent); + } + + protected virtual string TraceStateColumnName => "TraceState"; + protected virtual TraceState? GetTraceState(DbDataReader dr) + { + if (!TryGetOrdinal(dr, TraceStateColumnName, out var ordinal) || dr.IsDBNull(ordinal)) + { + return null; + } + + var traceState = dr.GetString(ordinal); + return string.IsNullOrEmpty(traceState) ? null : new TraceState(traceState); + } + + private static partial class Log + { + [LoggerMessage(LogLevel.Warning, "A duplicate was detected in the batch")] + public static partial void DuplicateDetectedInBatch(ILogger logger); + } } } diff --git a/src/Paramore.Brighter/RelationalDatabaseInbox.cs b/src/Paramore.Brighter/RelationalDatabaseInbox.cs index 2c62af7825..36a9a5ba44 100644 --- a/src/Paramore.Brighter/RelationalDatabaseInbox.cs +++ b/src/Paramore.Brighter/RelationalDatabaseInbox.cs @@ -28,22 +28,29 @@ THE SOFTWARE. */ using System.Data; using System.Data.Common; using System.Linq; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Paramore.Brighter.Inbox.Exceptions; +using Paramore.Brighter.JsonConverters; using Paramore.Brighter.Observability; namespace Paramore.Brighter { public abstract class RelationalDatabaseInbox( DbSystem dbSystem, - string databaseName, - string inboxTableName, + IAmARelationalDatabaseConfiguration configuration, + IAmARelationalDbConnectionProvider connectionProvider, IRelationalDatabaseInboxQueries queries, ILogger logger, InstrumentationOptions instrumentationOptions = InstrumentationOptions.All) : IAmAnInboxSync, IAmAnInboxAsync { + protected IAmARelationalDatabaseConfiguration DatabaseConfiguration { get; } = configuration; + + protected IAmARelationalDbConnectionProvider ConnectionProvider { get; } = connectionProvider; + /// public bool ContinueOnCapturedContext { get; set; } @@ -56,12 +63,13 @@ public void Add(T command, string contextKey, RequestContext? requestContext, { var dbAttributes = new Dictionary() { - {"db.operation.parameter.command.id", command.Id}, - {"db.operation.name", ExtractSqlOperationName(queries.AddCommand)}, - {"db.query.text", queries.AddCommand} + { "db.operation.parameter.command.id", command.Id }, + { "db.operation.name", ExtractSqlOperationName(queries.AddCommand) }, + { "db.query.text", queries.AddCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Add, inboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Add, + DatabaseConfiguration.InBoxTableName, dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -72,7 +80,8 @@ public void Add(T command, string contextKey, RequestContext? requestContext, connection => CreateAddCommand(connection, timeoutInMilliseconds, parameters), () => { - logger.LogWarning("Inbox: A duplicate command with the ID {Id} was inserted into the Inbox, ignoring and continuing", + logger.LogWarning( + "Inbox: A duplicate command with the ID {Id} was inserted into the Inbox, ignoring and continuing", command.Id); }); } @@ -86,14 +95,15 @@ public void Add(T command, string contextKey, RequestContext? requestContext, public T Get(string id, string contextKey, RequestContext? requestContext, int timeoutInMilliseconds) where T : class, IRequest { - var dbAttributes = new Dictionary() + var dbAttributes = new Dictionary { - {"db.operation.parameter.command.id", id}, - {"db.operation.name", ExtractSqlOperationName(queries.GetCommand)}, - {"db.query.text", queries.GetCommand} + { "db.operation.parameter.command.id", id }, + { "db.operation.name", ExtractSqlOperationName(queries.GetCommand) }, + { "db.query.text", queries.GetCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, inboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, + DatabaseConfiguration.InBoxTableName, dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -115,14 +125,15 @@ public T Get(string id, string contextKey, RequestContext? requestContext, in public bool Exists(string id, string contextKey, RequestContext? requestContext, int timeoutInMilliseconds) where T : class, IRequest { - var dbAttributes = new Dictionary() + var dbAttributes = new Dictionary { - {"db.operation.parameter.command.id", id}, - {"db.operation.name", ExtractSqlOperationName(queries.ExistsCommand)}, - {"db.query.text", queries.ExistsCommand} + { "db.operation.parameter.command.id", id }, + { "db.operation.name", ExtractSqlOperationName(queries.ExistsCommand) }, + { "db.query.text", queries.ExistsCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Exists, inboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Exists, + DatabaseConfiguration.InBoxTableName, dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -141,17 +152,19 @@ public bool Exists(string id, string contextKey, RequestContext? requestConte } /// - public async Task AddAsync(T command, string contextKey, RequestContext? requestContext, int timeoutInMilliseconds, CancellationToken cancellationToken) + public async Task AddAsync(T command, string contextKey, RequestContext? requestContext, + int timeoutInMilliseconds, CancellationToken cancellationToken) where T : class, IRequest { - var dbAttributes = new Dictionary() + var dbAttributes = new Dictionary { - {"db.operation.parameter.command.id", command.Id}, - {"db.operation.name", ExtractSqlOperationName(queries.AddCommand)}, - {"db.query.text", queries.AddCommand} + { "db.operation.parameter.command.id", command.Id }, + { "db.operation.name", ExtractSqlOperationName(queries.AddCommand) }, + { "db.query.text", queries.AddCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Add, inboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Add, + DatabaseConfiguration.InBoxTableName, dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -162,7 +175,8 @@ await WriteToStoreAsync( connection => CreateAddCommand(connection, timeoutInMilliseconds, parameters), () => { - logger.LogWarning("Inbox: A duplicate command with the ID {Id} was inserted into the Inbox, ignoring and continuing", + logger.LogWarning( + "Inbox: A duplicate command with the ID {Id} was inserted into the Inbox, ignoring and continuing", command.Id); }, cancellationToken); @@ -174,17 +188,19 @@ await WriteToStoreAsync( } /// - public async Task GetAsync(string id, string contextKey, RequestContext? requestContext, int timeoutInMilliseconds, CancellationToken cancellationToken) + public async Task GetAsync(string id, string contextKey, RequestContext? requestContext, + int timeoutInMilliseconds, CancellationToken cancellationToken) where T : class, IRequest { var dbAttributes = new Dictionary() { - {"db.operation.parameter.command.id", id}, - {"db.operation.name", ExtractSqlOperationName(queries.GetCommand)}, - {"db.query.text", queries.GetCommand} + { "db.operation.parameter.command.id", id }, + { "db.operation.name", ExtractSqlOperationName(queries.GetCommand) }, + { "db.query.text", queries.GetCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Get, inboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Get, + DatabaseConfiguration.InBoxTableName, dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -204,17 +220,19 @@ public async Task GetAsync(string id, string contextKey, RequestContext? r } /// - public async Task ExistsAsync(string id, string contextKey, RequestContext? requestContext, int timeoutInMilliseconds, CancellationToken cancellationToken) + public async Task ExistsAsync(string id, string contextKey, RequestContext? requestContext, + int timeoutInMilliseconds, CancellationToken cancellationToken) where T : class, IRequest { var dbAttributes = new Dictionary() { - {"db.operation.parameter.command.id", id}, - {"db.operation.name", ExtractSqlOperationName(queries.ExistsCommand)}, - {"db.query.text", queries.ExistsCommand} + { "db.operation.parameter.command.id", id }, + { "db.operation.name", ExtractSqlOperationName(queries.ExistsCommand) }, + { "db.query.text", queries.ExistsCommand } }; var span = Tracer?.CreateDbSpan( - new BoxSpanInfo(dbSystem, databaseName, BoxDbOperation.Exists, inboxTableName, dbAttributes: dbAttributes), + new BoxSpanInfo(dbSystem, DatabaseConfiguration.DatabaseName, BoxDbOperation.Exists, + DatabaseConfiguration.InBoxTableName, dbAttributes: dbAttributes), requestContext?.Span, options: instrumentationOptions); @@ -233,41 +251,134 @@ public async Task ExistsAsync(string id, string contextKey, RequestCont } } - protected abstract void WriteToStore( - Func commandFunc, - Action? loggingAction - ); + protected abstract bool IsExceptionUniqueOrDuplicateIssue(Exception ex); + + protected virtual void WriteToStore(Func commandFunc, Action? loggingAction) + { + var connection = GetOpenConnection(ConnectionProvider); - protected abstract Task WriteToStoreAsync( + using var command = commandFunc.Invoke(connection); + try + { + command.ExecuteNonQuery(); + } + catch (DbException exception) + { + if (!IsExceptionUniqueOrDuplicateIssue(exception)) + { + throw; + } + + loggingAction?.Invoke(); + } + finally + { + FinishWrite(connection); + } + } + + protected virtual async Task WriteToStoreAsync( Func commandFunc, Action? loggingAction, CancellationToken cancellationToken - ); + ) + { + var connection = await GetOpenConnectionAsync(ConnectionProvider, cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + +#if NETSTANDARD + using var command = commandFunc.Invoke(connection); +#else + await using var command = commandFunc.Invoke(connection); +#endif - protected abstract T ReadFromStore( + try + { + await command + .ExecuteNonQueryAsync(cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + } + catch (DbException exception) + { + if (!IsExceptionUniqueOrDuplicateIssue(exception)) + { + throw; + } + + loggingAction?.Invoke(); + } + finally + { + FinishWrite(connection); + } + } + + protected virtual T ReadFromStore( Func commandFunc, Func resultFunc, string commandId - ); + ) + { + var connection = GetOpenConnection(ConnectionProvider); + using var command = commandFunc.Invoke(connection); + try + { + return resultFunc.Invoke(command.ExecuteReader(), commandId); + } + finally + { + connection.Close(); + } + } - protected abstract Task ReadFromStoreAsync( + protected virtual async Task ReadFromStoreAsync( Func commandFunc, Func> resultFunc, string commandId, CancellationToken cancellationToken - ); + ) + { + var connection = await GetOpenConnectionAsync(ConnectionProvider, cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + +#if NETSTANDARD + using var command = commandFunc.Invoke(connection); +#else + await using var command = commandFunc.Invoke(connection); +#endif + try + { + var dr = await command.ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); - protected DbConnection GetOpenConnection(IAmARelationalDbConnectionProvider connectionProvider) + return await resultFunc.Invoke(dr, commandId, cancellationToken) + .ConfigureAwait(ContinueOnCapturedContext); + } + finally + { +#if NETSTANDARD + connection.Close(); +#else + await connection + .CloseAsync() + .ConfigureAwait(ContinueOnCapturedContext); +#endif + } + } + + protected static DbConnection GetOpenConnection(IAmARelationalDbConnectionProvider connectionProvider) { var connection = connectionProvider.GetConnection(); if (connection.State != ConnectionState.Open) + { connection.Open(); + } return connection; } - protected async Task GetOpenConnectionAsync( + protected static async Task GetOpenConnectionAsync( IAmARelationalDbConnectionProvider connectionProvider, CancellationToken cancellationToken) { var connection = await connectionProvider.GetConnectionAsync(cancellationToken); @@ -278,7 +389,7 @@ protected async Task GetOpenConnectionAsync( return connection; } - protected void FinishWrite(DbConnection connection) + protected static void FinishWrite(DbConnection connection) { connection.Close(); } @@ -293,24 +404,138 @@ private DbCommand CreateGetCommand(DbConnection connection, int inboxTimeout, ID => CreateCommand(connection, GenerateSqlText(queries.GetCommand), inboxTimeout, parameters); private string GenerateSqlText(string sqlFormat, params string[] orderedParams) - => string.Format(sqlFormat, orderedParams.Prepend(inboxTableName).ToArray()); + => string.Format(sqlFormat, orderedParams.Prepend(DatabaseConfiguration.InBoxTableName).ToArray()); - protected abstract DbCommand CreateCommand(DbConnection connection, string sqlText, int outBoxTimeout, - params IDbDataParameter[] parameters); + protected virtual DbCommand CreateCommand(DbConnection connection, string sqlText, int outBoxTimeout, + params IDbDataParameter[] parameters) + { - protected abstract IDbDataParameter[] CreateAddParameters(T command, string contextKey) where T : class, IRequest; + var command = connection.CreateCommand(); - protected abstract IDbDataParameter[] CreateExistsParameters(string commandId, string contextKey); + command.CommandTimeout = outBoxTimeout < 0 ? 0 : outBoxTimeout; + command.CommandText = sqlText; + command.Parameters.AddRange(parameters); - protected abstract IDbDataParameter[] CreateGetParameters(string commandId, string contextKey); + return command; + } - protected abstract T MapFunction(DbDataReader dr, string commandId) where T : class, IRequest; + protected abstract IDbDataParameter CreateSqlParameter(string parameterName, object? value); - protected abstract Task MapFunctionAsync(DbDataReader dr, string commandId, CancellationToken cancellationToken) where T : class, IRequest; + protected virtual IDbDataParameter[] CreateAddParameters(T command, string contextKey) + where T : class, IRequest + { + var commandJson = JsonSerializer.Serialize(command, JsonSerialisationOptions.Options); + return + [ + CreateSqlParameter("@CommandID", command.Id.Value), + CreateSqlParameter("@CommandType", typeof(T).Name), + CreateSqlParameter("@CommandBody", commandJson), + CreateSqlParameter("@Timestamp", DateTime.UtcNow), + CreateSqlParameter("@ContextKey", contextKey) + ]; + } + + protected virtual IDbDataParameter[] CreateExistsParameters(string commandId, string contextKey) + { + return + [ + CreateSqlParameter("@CommandID", commandId), + CreateSqlParameter("@ContextKey", contextKey) + ]; + } + + protected virtual IDbDataParameter[] CreateGetParameters(string commandId, string contextKey) + { + return + [ + CreateSqlParameter("@CommandID", commandId), + CreateSqlParameter("@ContextKey", contextKey) + ]; + } + + protected virtual string CommandBodyColumnName => "CommandBody"; + protected virtual T MapFunction(DbDataReader dr, string commandId) where T : class, IRequest + { + try + { + if (dr.Read()) + { + var body = dr.GetString(dr.GetOrdinal(CommandBodyColumnName)); + return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options)!; + } + } + finally + { + dr.Close(); + } + + throw new RequestNotFoundException(commandId); + } + + protected virtual async Task MapFunctionAsync(DbDataReader dr, string commandId, + CancellationToken cancellationToken) where T : class, IRequest + { + try + { + if (await dr.ReadAsync(cancellationToken).ConfigureAwait(ContinueOnCapturedContext)) + { + var body = dr.GetString(dr.GetOrdinal(CommandBodyColumnName)); + return JsonSerializer.Deserialize(body, JsonSerialisationOptions.Options)!; + } + } + finally + { + +#if NETSTANDARD + dr.Close(); +#else + await dr + .CloseAsync() + .ConfigureAwait(ContinueOnCapturedContext); +#endif + } - protected abstract bool MapBoolFunction(DbDataReader dr, string commandId); + throw new RequestNotFoundException(commandId); + } - protected abstract Task MapBoolFunctionAsync(DbDataReader dr, string commandId, CancellationToken cancellationToken); + protected virtual bool MapBoolFunction(DbDataReader dr, string commandId) + { + try + { + return dr.HasRows; + } + finally + { + dr.Close(); + } + } + +#if NETSTANDARD + protected virtual Task MapBoolFunctionAsync(DbDataReader dr, string commandId, CancellationToken cancellationToken) + { + try + { + return Task.FromResult(dr.HasRows); + } + finally + { + dr.Close(); + } + } +#else + protected virtual async Task MapBoolFunctionAsync(DbDataReader dr, string commandId, + CancellationToken cancellationToken) + { + try + { + return dr.HasRows; + } + finally + { + await dr.CloseAsync().ConfigureAwait(ContinueOnCapturedContext); + } + } +#endif private static string ExtractSqlOperationName(string queryText) { From 1939ab56a22661e02a0746eae3db3b319691f5f4 Mon Sep 17 00:00:00 2001 From: Rafael Andrade Date: Fri, 11 Jul 2025 14:48:08 +0100 Subject: [PATCH 2/3] fix: MySQL tests --- .../MySqlOutbox.cs | 2 +- .../MySqlOutboxBuilder.cs | 102 +++++++++--------- .../RelationDatabaseOutbox.cs | 6 +- ...iting_a_message_to_a_binary_body_outbox.cs | 6 +- 4 files changed, 61 insertions(+), 55 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs b/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs index b9fbe7d34b..e29b7c7091 100644 --- a/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs +++ b/src/Paramore.Brighter.Outbox.MySql/MySqlOutbox.cs @@ -70,7 +70,7 @@ protected override IDbDataParameter CreateSqlParameter(string parameterName, obj /// protected override IDbDataParameter CreateSqlParameter(string parameterName, DbType dbType, object? value) { - return new MySqlParameter { ParameterName = parameterName, Value = value, DbType = dbType }; + return new MySqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value, DbType = dbType }; } /// diff --git a/src/Paramore.Brighter.Outbox.MySql/MySqlOutboxBuilder.cs b/src/Paramore.Brighter.Outbox.MySql/MySqlOutboxBuilder.cs index 672cb4fb92..8b423521e3 100644 --- a/src/Paramore.Brighter.Outbox.MySql/MySqlOutboxBuilder.cs +++ b/src/Paramore.Brighter.Outbox.MySql/MySqlOutboxBuilder.cs @@ -33,55 +33,61 @@ namespace Paramore.Brighter.Outbox.MySql /// public class MySqlOutboxBuilder { - const string TextOutboxDdl = @"CREATE TABLE {0} ( - `MessageId`VARCHAR(255) NOT NULL , - `Topic` VARCHAR(255) NOT NULL , - `MessageType` VARCHAR(32) NOT NULL , - `Timestamp` TIMESTAMP(3) NOT NULL , - `CorrelationId`VARCHAR(255) NULL , - `ReplyTo` VARCHAR(255) NULL , - `ContentType` VARCHAR(128) NULL , - `PartitionKey` VARCHAR(128) NULL , - `Dispatched` TIMESTAMP(3) NULL , - `HeaderBag` TEXT NOT NULL , - `Body` TEXT NOT NULL , - `Source` VARCHAR(255) NULL, - `Type` VARCHAR(255) NULL, - `DataSchema` VARCHAR(255) NULL, - `Subject` VARCHAR(255) NULL, - `TraceParent` VARCHAR(255) NULL, - `TraceState` VARCHAR(255) NULL, - `Baggage` TEXT NULL, - `Created` TIMESTAMP(3) NOT NULL DEFAULT NOW(3), - `CreatedID` INT(11) NOT NULL AUTO_INCREMENT, - UNIQUE(`CreatedID`), - PRIMARY KEY (`MessageId`) -) ENGINE = InnoDB;"; + const string TextOutboxDdl = + """ + CREATE TABLE {0} ( + `MessageId`VARCHAR(255) NOT NULL , + `Topic` VARCHAR(255) NOT NULL , + `MessageType` VARCHAR(32) NOT NULL , + `Timestamp` TIMESTAMP(3) NOT NULL , + `CorrelationId`VARCHAR(255) NULL , + `ReplyTo` VARCHAR(255) NULL , + `ContentType` VARCHAR(128) NULL , + `PartitionKey` VARCHAR(128) NULL , + `Dispatched` TIMESTAMP(3) NULL , + `HeaderBag` TEXT NOT NULL , + `Body` TEXT NOT NULL , + `Source` VARCHAR(255) NULL, + `Type` VARCHAR(255) NULL, + `DataSchema` VARCHAR(255) NULL, + `Subject` VARCHAR(255) NULL, + `TraceParent` VARCHAR(255) NULL, + `TraceState` VARCHAR(255) NULL, + `Baggage` TEXT NULL, + `Created` TIMESTAMP(3) NOT NULL DEFAULT NOW(3), + `CreatedID` INT(11) NOT NULL AUTO_INCREMENT, + UNIQUE(`CreatedID`), + PRIMARY KEY (`MessageId`) + ) ENGINE = InnoDB; + """; - const string BinaryOutboxDdl = @"CREATE TABLE {0} ( - `MessageId` VARCHAR(255) NOT NULL , - `Topic` VARCHAR(255) NOT NULL , - `MessageType` VARCHAR(32) NOT NULL , - `Timestamp` TIMESTAMP(3) NOT NULL , - `CorrelationId` VARCHAR(255) NULL , - `ReplyTo` VARCHAR(255) NULL , - `ContentType` VARCHAR(128) NULL , - `PartitionKey` VARCHAR(128) NULL , - `Dispatched` TIMESTAMP(3) NULL , - `HeaderBag` TEXT NOT NULL , - `Body` BLOB NOT NULL , - `Source` VARCHAR(255) NULL, - `Type` VARCHAR(255) NULL, - `DataSchema` VARCHAR(255) NULL, - `Subject` VARCHAR(255) NULL, - `TraceParent` VARCHAR(255) NULL, - `TraceState` VARCHAR(255) NULL, - `Baggage` TEXT NULL, - `Created` TIMESTAMP(3) NOT NULL DEFAULT NOW(3), - `CreatedID` INT(11) NOT NULL AUTO_INCREMENT, - UNIQUE(`CreatedID`), - PRIMARY KEY (`MessageId`) -) ENGINE = InnoDB;"; + const string BinaryOutboxDdl = + """ + CREATE TABLE {0} ( + `MessageId` VARCHAR(255) NOT NULL , + `Topic` VARCHAR(255) NOT NULL , + `MessageType` VARCHAR(32) NOT NULL , + `Timestamp` TIMESTAMP(3) NOT NULL , + `CorrelationId` VARCHAR(255) NULL , + `ReplyTo` VARCHAR(255) NULL , + `ContentType` VARCHAR(128) NULL , + `PartitionKey` VARCHAR(128) NULL , + `Dispatched` TIMESTAMP(3) NULL , + `HeaderBag` TEXT NOT NULL , + `Body` BLOB NOT NULL , + `Source` VARCHAR(255) NULL, + `Type` VARCHAR(255) NULL, + `DataSchema` VARCHAR(255) NULL, + `Subject` VARCHAR(255) NULL, + `TraceParent` VARCHAR(255) NULL, + `TraceState` VARCHAR(255) NULL, + `Baggage` TEXT NULL, + `Created` TIMESTAMP(3) NOT NULL DEFAULT NOW(3), + `CreatedID` INT(11) NOT NULL AUTO_INCREMENT, + UNIQUE(`CreatedID`), + PRIMARY KEY (`MessageId`) + ) ENGINE = InnoDB; + """; const string outboxExistsQuery = @"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = '{0}') AS TableExists;"; diff --git a/src/Paramore.Brighter/RelationDatabaseOutbox.cs b/src/Paramore.Brighter/RelationDatabaseOutbox.cs index aad9283506..76e9dc720b 100644 --- a/src/Paramore.Brighter/RelationDatabaseOutbox.cs +++ b/src/Paramore.Brighter/RelationDatabaseOutbox.cs @@ -1256,7 +1256,7 @@ protected virtual IDbDataParameter[] InitAddDbParameters(Message message, int? p var bagJson = JsonSerializer.Serialize(message.Header.Bag, JsonSerialisationOptions.Options); var body = DatabaseConfiguration.BinaryMessagePayload ? - CreateSqlParameter($"@{prefix}Body", DbType.Byte, message.Body.Bytes) + CreateSqlParameter($"@{prefix}Body", DbType.Binary, message.Body.Bytes) : CreateSqlParameter($"@{prefix}Body", DbType.String, message.Body.Value); return @@ -1502,8 +1502,8 @@ protected virtual ContentType GetContentType(DbDataReader dr) return new ContentType(MediaTypeNames.Text.Plain); } - var replyTo = dr.GetString(ordinal); - return string.IsNullOrEmpty(replyTo) ? new ContentType(MediaTypeNames.Text.Plain) : new ContentType(replyTo); + var contentType = dr.GetString(ordinal); + return string.IsNullOrEmpty(contentType) ? new ContentType(MediaTypeNames.Text.Plain) : new ContentType(contentType); } protected virtual string HeaderBagColumnName => "HeaderBag"; diff --git a/tests/Paramore.Brighter.MySQL.Tests/Outbox/When_writing_a_message_to_a_binary_body_outbox.cs b/tests/Paramore.Brighter.MySQL.Tests/Outbox/When_writing_a_message_to_a_binary_body_outbox.cs index 8b87017bab..d5bb7dfde4 100644 --- a/tests/Paramore.Brighter.MySQL.Tests/Outbox/When_writing_a_message_to_a_binary_body_outbox.cs +++ b/tests/Paramore.Brighter.MySQL.Tests/Outbox/When_writing_a_message_to_a_binary_body_outbox.cs @@ -3,7 +3,7 @@ using Paramore.Brighter.Outbox.MySql; using Xunit; -namespace Paramore.Brighter.MySQL.Tests +namespace Paramore.Brighter.MySQL.Tests.Outbox { public class MySqlOutboxWritingBinaryMessageTests { @@ -46,7 +46,7 @@ public MySqlOutboxWritingBinaryMessageTests() _messageEarliest = new Message( messageHeader, - new MessageBody(new byte[] { 1, 2, 3, 4, 5 }, new ContentType(MediaTypeNames.Application.Octet), CharacterEncoding.Raw ) + new MessageBody([1, 2, 3, 4, 5], new ContentType(MediaTypeNames.Application.Octet), CharacterEncoding.Raw ) ); _mySqlOutbox.Add(_messageEarliest, new RequestContext()); } @@ -57,7 +57,7 @@ public void When_writing_a_message_to_a_binary_body_outbox() _storedMessage = _mySqlOutbox.Get(_messageEarliest.Id, new RequestContext()); //should read the message from the sql outbox - Assert.Equal(_messageEarliest.Body.Value, _storedMessage.Body.Value); + Assert.Equal(_messageEarliest.Body.Bytes, _storedMessage.Body.Bytes); //should read the header from the sql outbox Assert.Equal(_messageEarliest.Header.Topic, _storedMessage.Header.Topic); From ff3b36473b3d810e53fb0827f056fcd0f1283c57 Mon Sep 17 00:00:00 2001 From: Rafael Andrade Date: Fri, 11 Jul 2025 15:05:14 +0100 Subject: [PATCH 3/3] fix: MSSQL unit test --- .../MsSqlOutbox.cs | 34 +++++++++++++------ .../RelationDatabaseOutbox.cs | 16 ++++----- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs b/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs index 59e719aa9c..c7ddae1b70 100644 --- a/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs +++ b/src/Paramore.Brighter.Outbox.MsSql/MsSqlOutbox.cs @@ -80,16 +80,30 @@ protected override IDbDataParameter CreateSqlParameter(string parameterName, DbT return new SqlParameter { ParameterName = parameterName, Value = value ?? DBNull.Value, DbType = dbType }; } - /// - protected override DateTimeOffset GetTimeStamp(DbDataReader dr) + protected override IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan since, int pageSize, int pageNumber) + { + var parameters = new IDbDataParameter[3]; + parameters[0] = new SqlParameter { ParameterName = "PageNumber", Value = pageNumber }; + parameters[1] = new SqlParameter { ParameterName = "PageSize", Value = pageSize }; + parameters[2] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(since)); + return parameters; + } + + protected override IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan dispatchedSince, int pageSize, int pageNumber) { - if (!TryGetOrdinal(dr, TimestampColumnName, out var ordinal) || dr.IsDBNull(ordinal)) - { - return DateTimeOffset.MinValue; - } - - var reader = (SqlDataReader)dr; - var dataTime = reader.GetDateTimeOffset(ordinal); - return dataTime; + var parameters = new IDbDataParameter[3]; + parameters[0] = new SqlParameter { ParameterName = "PageNumber", Value = pageNumber }; + parameters[1] = new SqlParameter { ParameterName = "PageSize", Value = pageSize }; + parameters[2] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); + + return parameters; } + + protected override IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber) + { + var parameters = new IDbDataParameter[2]; + parameters[0] = new SqlParameter { ParameterName = "PageNumber", Value = pageNumber }; + parameters[1] = new SqlParameter { ParameterName = "PageSize", Value = pageSize }; + return parameters; + } } diff --git a/src/Paramore.Brighter/RelationDatabaseOutbox.cs b/src/Paramore.Brighter/RelationDatabaseOutbox.cs index 76e9dc720b..fd04ad0bbf 100644 --- a/src/Paramore.Brighter/RelationDatabaseOutbox.cs +++ b/src/Paramore.Brighter/RelationDatabaseOutbox.cs @@ -1220,9 +1220,9 @@ protected virtual IDbDataParameter[] CreatePagedOutstandingParameters(TimeSpan s int pageNumber) { var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("TimestampSince", DateTimeOffset.UtcNow.Subtract(since)); + parameters[0] = CreateSqlParameter("@Skip", Math.Max(pageNumber - 1, 0) * pageSize); + parameters[1] = CreateSqlParameter("@Take", pageSize); + parameters[2] = CreateSqlParameter("@TimestampSince", DateTimeOffset.UtcNow.Subtract(since)); return parameters; } @@ -1231,9 +1231,9 @@ protected virtual IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan di { var parameters = new IDbDataParameter[3]; - parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - parameters[1] = CreateSqlParameter("Take", pageSize); - parameters[2] = CreateSqlParameter("DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); + parameters[0] = CreateSqlParameter("@Skip", Math.Max(pageNumber - 1, 0) * pageSize); + parameters[1] = CreateSqlParameter("@Take", pageSize); + parameters[2] = CreateSqlParameter("@DispatchedSince", DateTimeOffset.UtcNow.Subtract(dispatchedSince)); return parameters; } @@ -1241,8 +1241,8 @@ protected virtual IDbDataParameter[] CreatePagedDispatchedParameters(TimeSpan di protected virtual IDbDataParameter[] CreatePagedReadParameters(int pageSize, int pageNumber) { var parameters = new IDbDataParameter[2]; - parameters[0] = CreateSqlParameter("Skip", Math.Max(pageNumber - 1, 0) * pageSize); - parameters[1] = CreateSqlParameter("Take", pageSize); + parameters[0] = CreateSqlParameter("@Skip", Math.Max(pageNumber - 1, 0) * pageSize); + parameters[1] = CreateSqlParameter("@Take", pageSize); return parameters; }