diff --git a/Foundatio.RabbitMQ.slnx b/Foundatio.RabbitMQ.slnx index 32228acd..a4aa7587 100644 --- a/Foundatio.RabbitMQ.slnx +++ b/Foundatio.RabbitMQ.slnx @@ -14,6 +14,7 @@ + diff --git a/src/Foundatio.RabbitMQ/Messaging/DeadLetterStrategy.cs b/src/Foundatio.RabbitMQ/Messaging/DeadLetterStrategy.cs index 3e21a87d..ae55e34c 100644 --- a/src/Foundatio.RabbitMQ/Messaging/DeadLetterStrategy.cs +++ b/src/Foundatio.RabbitMQ/Messaging/DeadLetterStrategy.cs @@ -18,7 +18,9 @@ public enum DeadLetterStrategy AtMostOnce, /// - /// Guarantees message transfer to the dead-letter exchange using internal publisher confirms. + /// Retains broker-managed dead letters until target queues confirm acceptance; duplicates remain possible. + /// Applies only to source quorum queues, not provider-managed terminal handoffs. + /// Destination durability and availability remain separate requirements. /// Requires overflow to be set to reject-publish (not drop-head). /// Uses more memory and CPU. Only enable when dead-lettered messages must not be lost. /// diff --git a/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBus.cs b/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBus.cs index 10aaf311..20348f75 100644 --- a/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBus.cs +++ b/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBus.cs @@ -1,13 +1,12 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Foundatio.AsyncEx; using Foundatio.Extensions; +using Foundatio.Serializer; using Foundatio.Utility; using Microsoft.Extensions.Logging; using RabbitMQ.Client; @@ -18,12 +17,11 @@ namespace Foundatio.Messaging; public class RabbitMQMessageBus : MessageBusBase { - private const string XDeliveryCountHeader = "x-delivery-count"; - private const string XOriginalMessageIdHeader = "x-original-message-id"; - private const string PriorityPropertyKey = "Priority"; - private static readonly Version _delayedExchangePluginIncompatibleVersion = new(4, 3); - private static readonly Version _globalQosRemovedVersion = new(4, 3); - + private static readonly Func _deliveryScope = + LoggerMessage.DefineScope("Message {MessageId}, delivery {DeliveryTag}"); + private readonly CancellationToken _shutdownToken; + private readonly RabbitMQTopology _topology; + private readonly Func _deserializeMessageBody; private readonly AsyncLock _lock = new(); private readonly AsyncManualResetEvent _publisherReady = new(true); private readonly ConnectionFactory _factory; @@ -33,44 +31,49 @@ public class RabbitMQMessageBus : MessageBusBase private volatile IChannel? _publisherChannel; private volatile IChannel? _subscriberChannel; private AsyncEventingBasicConsumer? _consumer; - private bool? _delayedExchangePluginEnabled; - private Version? _serverVersion; - private readonly bool _isQuorumQueue; private volatile bool _isPublisherBlocked; private volatile string? _publisherBlockedReason; + private readonly AsyncLocal _initializingSubscriber = new(); + private readonly AsyncLocal _deliveryCancellation = new(); + private readonly AsyncLock _subscriberLock = new(); + private readonly CancellationTokenSource _shutdown = new(); + private readonly ConcurrentDictionary _subscriberRegistrations = new(); + private readonly object _maintenanceSync = new(); + private readonly AsyncAutoResetEvent _subscriptionChanged = new(); + private Task? _maintenanceTask; + private Task? _shutdownCancellation; + private DeliveryEpoch _deliveryEpoch; + private volatile bool _subscriberRecovering; + private volatile bool _permanentSubscriberFault; + private volatile Exception? _subscriptionError; + private volatile string? _subscriptionQueueName; + private readonly AsyncLock _handoffLock = new(); + private IConnection? _handoffConnection; + private IChannel? _handoffChannel; + private int _blockedDeliveries; + private int _activeDeliveries; + private volatile Exception? _deliveryError; public RabbitMQMessageBus(RabbitMQMessageBusOptions options) : base(options) { - ArgumentException.ThrowIfNullOrWhiteSpace(options?.ConnectionString, nameof(options.ConnectionString)); - + ArgumentException.ThrowIfNullOrWhiteSpace(options.ConnectionString, nameof(options.ConnectionString)); if (!Uri.TryCreate(options.ConnectionString, UriKind.Absolute, out var primaryUri)) - throw new ArgumentException("ConnectionString is not a valid URI."); - + throw new ArgumentException("ConnectionString is not a valid URI.", nameof(options.ConnectionString)); if (!primaryUri.Scheme.Equals("amqp", StringComparison.OrdinalIgnoreCase) && !primaryUri.Scheme.Equals("amqps", StringComparison.OrdinalIgnoreCase)) - throw new ArgumentException($"ConnectionString must use amqp:// or amqps:// scheme: {SanitizeUri(primaryUri)}"); - - _isQuorumQueue = RabbitMQMessageBusOptions.IsQuorumQueue(options.Arguments); - if (_isQuorumQueue && options.MaxPriority.HasValue) - throw new InvalidOperationException("MaxPriority applies only to classic queues and cannot be used with quorum queues."); - - // Initialize the connection factory with credentials/vhost from connection string - // Automatic recovery will allow the connections to be restored in case the server is - // restarted or there has been any network failures. TopologyRecoveryEnabled is already - // enabled by default. NetworkRecoveryInterval is also by default set to 5 seconds. - _factory = new ConnectionFactory - { - Uri = primaryUri, - AutomaticRecoveryEnabled = true - }; + throw new ArgumentException("ConnectionString must use amqp:// or amqps://.", nameof(options.ConnectionString)); + RabbitMQMessageBusOptions.Validate(options); + _topology = new RabbitMQTopology(options, _logger); + _deserializeMessageBody = DeserializeMessageBody; + _factory = new ConnectionFactory { Uri = primaryUri, AutomaticRecoveryEnabled = true }; if (options.RequestedHeartbeat.HasValue) _factory.RequestedHeartbeat = options.RequestedHeartbeat.Value; - if (options.NetworkRecoveryInterval.HasValue) _factory.NetworkRecoveryInterval = options.NetworkRecoveryInterval.Value; - _endpoints = RabbitMQEndpointResolver.CreateEndpoints(_factory, options.Hosts); + _shutdownToken = _shutdown.Token; + _deliveryEpoch = new DeliveryEpoch(_shutdownToken); } public RabbitMQMessageBus(Builder config) @@ -78,468 +81,853 @@ public RabbitMQMessageBus(BuilderWhether a live transport consumer can dispatch to registered handlers without a retained-delivery blockage. + public bool IsSubscriptionReady => !IsDisposed && !_subscriberRecovering && !_permanentSubscriberFault + && _subscriptionError is null && Volatile.Read(ref _blockedDeliveries) == 0 && !_subscribers.IsEmpty + && _subscriberConnection is { IsOpen: true } && _subscriberChannel is { IsOpen: true } && _consumer is { IsRunning: true }; + + /// The latest subscription initialization/recovery error, cleared after successful recovery. + public Exception? LastSubscriptionError => _subscriptionError; + + /// + /// The latest retained-delivery or handoff error. Retention is not successful processing. + /// Inspect IsSubscriptionReady as well; this is not a broker queue-depth measurement. + /// + public Exception? LastDeliveryError => _deliveryError; + + // Test synchronization observes callback completion, not just entry into the user handler. + internal int ActiveDeliveryCount => Volatile.Read(ref _activeDeliveries); + + protected override Task RemoveTopicSubscriptionAsync() => CleanupTransportAsync(_subscriberLock, async () => + { + await ClearSubscriberChannelAsync().AnyContext(); + await ClearSubscriberConnectionAsync().AnyContext(); + }, "subscriber"); + + protected override async Task ShutdownAsync() { - await CloseSubscriberConnectionAsync().AnyContext(); + _shutdownCancellation = _shutdown.CancelAsync(); + try + { + await _shutdownCancellation.WaitAsync(_options.ShutdownTimeout).AnyContext(); + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + _logger.LogWarning(exception, "Subscription cancellation did not finish cleanly"); + } + InvalidateDeliveries(); + if (_maintenanceTask is not null) + { + try + { + await _maintenanceTask.WaitAsync(_options.ShutdownTimeout).AnyContext(); + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + _logger.LogWarning(exception, "Subscription maintenance did not stop cleanly"); + } + } + await base.ShutdownAsync().AnyContext(); } protected override async Task CleanupAsync() { _factory.AutomaticRecoveryEnabled = false; - - await ClosePublisherConnectionAsync().AnyContext(); - await CloseSubscriberConnectionAsync().AnyContext(); - + await CleanupTransportAsync(_lock, ClearPublisherTransportAsync, "publisher").AnyContext(); + await CleanupTransportAsync(_handoffLock, ClearHandoffTransportAsync, "handoff").AnyContext(); + foreach (var registration in _subscriberRegistrations.Values) + registration.Unregister(); + _subscriberRegistrations.Clear(); _publisherReady.Set(); + _deliveryEpoch.Cancel(_logger); + // Cancellation callbacks can be user code. Do not dispose their source while + // they are still running, or wait forever for them on the transport cleanup path. + _ = FinishShutdownCancellationAsync(); } - protected override async Task EnsureTopicSubscriptionAsync(CancellationToken cancellationToken) + private async Task CleanupTransportAsync(AsyncLock mutex, Func cleanup, string role) { - if (_subscriberChannel is not null) - return; + // Keep ownership of cleanup after the caller's deadline. A busy transport must + // eventually be disposed without making application shutdown wait for its lock. + var pending = CleanupWhenAvailableAsync(); + try + { + await pending.WaitAsync(_options.ShutdownTimeout).AnyContext(); + } + catch (TimeoutException) + { + _logger.LogWarning("Transport cleanup continues after the shutdown wait ({Role})", role); + } - await EnsureTopicCreatedAsync(cancellationToken).AnyContext(); + async Task CleanupWhenAvailableAsync() + { + try + { + using (await mutex.LockAsync().AnyContext()) + await cleanup().AnyContext(); + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + _logger.LogWarning(exception, "Deferred transport cleanup failed ({Role})", role); + } + } + } - using (await _lock.LockAsync(cancellationToken).AnyContext()) + private async Task FinishShutdownCancellationAsync() + { + try { - if (_subscriberChannel is not null) - return; + if (_shutdownCancellation is not null) + await _shutdownCancellation.AnyContext(); + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + _logger.LogWarning(exception, "A shutdown cancellation callback failed"); + } + finally + { + _shutdown.Dispose(); + } + } - _subscriberConnection = await CreateConnectionAsync().AnyContext(); - DetectServerVersion(_subscriberConnection); - RegisterSubscriberConnectionEventHandlers(); + protected override Task SubscribeImplAsync(Func handler, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(handler); + var subscriber = new Subscriber + { + Type = typeof(T), + CancellationToken = cancellationToken, + Action = async (message, token) => + { + if (message is not T typed) + { + if (_options.RequireSuccessfulDispatch) + throw new InvalidDeliveryException("The delivery cannot be assigned to the required handler."); + return; + } + var deliveryToken = _options.AcknowledgementStrategy == AcknowledgementStrategy.Automatic && !_options.RequireSuccessfulDispatch + ? _deliveryCancellation.Value : default; + using var linked = CancellationTokenSource.CreateLinkedTokenSource(token, + deliveryToken.CanBeCanceled ? deliveryToken : _shutdownToken); + linked.Token.ThrowIfCancellationRequested(); + await handler(typed, linked.Token).WaitAsync(linked.Token).AnyContext(); + } + }; + if (typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(IMessage<>)) + subscriber.GenericType = typeof(Message<>).MakeGenericType(typeof(T).GenericTypeArguments[0]); + if (!_subscribers.TryAdd(subscriber.Id, subscriber)) + throw new MessageBusException("Unable to register the local subscription."); + + var registration = cancellationToken.Register(() => + { + _subscribers.TryRemove(subscriber.Id, out _); + if (_subscriberRegistrations.TryRemove(subscriber.Id, out var current)) + current.Unregister(); + _subscriptionChanged.Set(); + }); + _subscriberRegistrations[subscriber.Id] = registration; + _permanentSubscriberFault = false; + lock (_maintenanceSync) + _maintenanceTask ??= MaintainSubscriptionAsync(); + // The base calls EnsureTopicSubscriptionAsync next; keep this method synchronous + // so that call receives its own subscriber context even with concurrent subscriptions. + _initializingSubscriber.Value = subscriber; + return Task.CompletedTask; + } - _subscriberChannel = await _subscriberConnection.CreateChannelAsync(cancellationToken: cancellationToken).AnyContext(); + protected override async Task EnsureTopicSubscriptionAsync(CancellationToken cancellationToken) + { + var subscriber = _initializingSubscriber.Value; + _initializingSubscriber.Value = null; + if (subscriber is null) + { + await InitializeSubscriptionAsync(cancellationToken).AnyContext(); + return; + } - // If InitPublisher is called first, then we will never come in this if-clause. - var delayedExchangeResult = await CreateDelayedExchangeAsync(_subscriberChannel).AnyContext(); - if (delayedExchangeResult is null) + var callerToken = subscriber.CancellationToken; + try + { + callerToken.ThrowIfCancellationRequested(); + // One caller can stop waiting without cancelling setup needed by other subscribers. + // Maintenance is already running so abandoned initialization cannot leave an orphan consumer. + var initialization = InitializeSubscriptionAsync(cancellationToken); + try { - await CreateRegularExchangeAsync(_subscriberChannel).AnyContext(); + await initialization.WaitAsync(callerToken).AnyContext(); } - else if (delayedExchangeResult is false) + catch (OperationCanceledException) when (callerToken.IsCancellationRequested) { - await _subscriberChannel.DisposeAsync().AnyContext(); - UnregisterSubscriberConnectionEventHandlers(); - await _subscriberConnection.DisposeAsync().AnyContext(); - - _subscriberConnection = await CreateConnectionAsync().AnyContext(); - DetectServerVersion(_subscriberConnection); - RegisterSubscriberConnectionEventHandlers(); - - _subscriberChannel = await _subscriberConnection.CreateChannelAsync(cancellationToken: cancellationToken).AnyContext(); - await CreateRegularExchangeAsync(_subscriberChannel).AnyContext(); + _ = ObserveSubscriptionInitializationAsync(initialization); + throw; } + callerToken.ThrowIfCancellationRequested(); + } + catch + { + _subscribers.TryRemove(subscriber.Id, out _); + if (_subscriberRegistrations.TryRemove(subscriber.Id, out var registration)) + registration.Unregister(); + _subscriptionChanged.Set(); + throw; + } + } - string queueName = await CreateQueueAsync(_subscriberChannel).AnyContext(); + private async Task ObserveSubscriptionInitializationAsync(Task initialization) + { + try + { + await initialization.AnyContext(); + } + catch (OperationCanceledException) when (_shutdown.IsCancellationRequested || IsDisposed) + { + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + _logger.LogDebug(exception, "Subscription initialization ended after its caller stopped waiting"); + } + } - // Set QoS (Quality of Service) settings for the consumer - if (_options.PrefetchCount > 0 || _options.PrefetchSize > 0) + private async Task InitializeSubscriptionAsync(CancellationToken cancellationToken) + { + if (_subscribers.IsEmpty) + return; + using var setup = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownToken); + setup.CancelAfter(TimeSpan.FromSeconds(30)); + using (await _subscriberLock.LockAsync(setup.Token).AnyContext()) + { + if (_subscriberChannel is { IsOpen: true } && _consumer is { IsRunning: true }) + return; + if (_subscriberRecovering) + return; + setup.Token.ThrowIfCancellationRequested(); + if (IsDisposed) + throw new MessageBusException("Cannot initialize a disposed subscription."); + try { -#pragma warning disable CS0618 // GlobalQos is obsolete but we still need to read it for backward compatibility - bool useGlobalQos = _options.GlobalQos; -#pragma warning restore CS0618 - if (useGlobalQos && _isQuorumQueue) + await EnsureTopicCreatedAsync(setup.Token).AnyContext(); + await ClearSubscriberChannelAsync().AnyContext(); + if (_subscribers.IsEmpty) + return; + if (_subscriberConnection is not { IsOpen: true }) { - _logger.LogWarning("GlobalQos is not supported on quorum queues. Falling back to per-channel prefetch (global: false). Remove the GlobalQos option to suppress this warning"); - useGlobalQos = false; + await ClearSubscriberConnectionAsync().AnyContext(); + _subscriberConnection = await CreateConnectionAsync(setup.Token).AnyContext(); + RegisterSubscriberConnectionEventHandlers(); } - else if (useGlobalQos && _serverVersion is not null && _serverVersion >= _globalQosRemovedVersion) + var serverVersion = _topology.DetectServerVersion(_subscriberConnection); + _subscriberChannel = await _subscriberConnection.CreateChannelAsync(cancellationToken: setup.Token).AnyContext(); + _subscriberChannel.ChannelShutdownAsync += OnSubscriberChannelShutdownAsync; + await _topology.DeclareSubscriptionExchangeAsync(_subscriberChannel, setup.Token).AnyContext(); + _subscriptionQueueName = await _topology.CreateQueueAsync(_subscriberChannel, serverVersion, setup.Token).AnyContext(); + await _topology.ConfigurePrefetchAsync(_subscriberChannel, serverVersion, setup.Token).AnyContext(); + if (_subscribers.IsEmpty) { - _logger.LogWarning("GlobalQos is not supported on RabbitMQ {ServerVersion}. Falling back to per-channel prefetch (global: false). Remove the GlobalQos option to suppress this warning", _serverVersion); - useGlobalQos = false; + await ClearSubscriberChannelAsync().AnyContext(); + return; } - else if (useGlobalQos && _serverVersion is not null) + _consumer = new AsyncEventingBasicConsumer(_subscriberChannel); + _consumer.ReceivedAsync += OnMessageAsync; + _consumer.ShutdownAsync += OnConsumerShutdownAsync; + _consumer.UnregisteredAsync += OnConsumerUnregisteredAsync; + var registered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task OnRegisteredAsync(object sender, ConsumerEventArgs args) { - _logger.LogWarning("GlobalQos is deprecated in RabbitMQ 4.3+ and will be removed in a future version. Use per-channel prefetch instead"); + registered.TrySetResult(); + return Task.CompletedTask; } - - await _subscriberChannel.BasicQosAsync(_options.PrefetchSize, _options.PrefetchCount, useGlobalQos, cancellationToken).AnyContext(); - _logger.LogDebug("Set channel QoS - PrefetchCount: {PrefetchCount}, PrefetchSize: {PrefetchSize}, Global: {GlobalQos} for acknowledgment strategy {AcknowledgementStrategy}", - _options.PrefetchCount, _options.PrefetchSize, useGlobalQos, _options.AcknowledgementStrategy); + _consumer.RegisteredAsync += OnRegisteredAsync; + try + { + await _subscriberChannel.BasicConsumeAsync(_subscriptionQueueName, + _options.AcknowledgementStrategy == AcknowledgementStrategy.FireAndForget, _consumer, cancellationToken: setup.Token).AnyContext(); + // The consume RPC completes before the dispatcher marks the consumer as running. + await registered.Task.WaitAsync(setup.Token).AnyContext(); + } + finally + { + _consumer.RegisteredAsync -= OnRegisteredAsync; + } + _subscriptionError = null; + _permanentSubscriberFault = false; } - else + catch (Exception exception) { - _logger.LogDebug("Using unlimited prefetch for acknowledgment strategy {AcknowledgementStrategy}", _options.AcknowledgementStrategy); + _subscriptionError = exception; + _permanentSubscriberFault = IsPermanentSubscriptionError(exception); + await ClearSubscriberChannelAsync().AnyContext(); + await ClearSubscriberConnectionAsync().AnyContext(); + throw; } - - _consumer = new AsyncEventingBasicConsumer(_subscriberChannel); - RegisterConsumerEventHandlers(); - - await _subscriberChannel.BasicConsumeAsync(queueName, _options.AcknowledgementStrategy == AcknowledgementStrategy.FireAndForget, _consumer, cancellationToken: cancellationToken).AnyContext(); - _logger.LogTrace("The unique channel number for the subscriber is : {ChannelNumber}", _subscriberChannel.ChannelNumber); } } - private Task OnSubscriberConnectionOnCallbackExceptionAsync(object sender, CallbackExceptionEventArgs e) + private async Task MaintainSubscriptionAsync() { - _logger.LogError(e.Exception, "Subscriber callback exception: {Message}", e.Exception.Message); - return Task.CompletedTask; + try + { + while (!_shutdown.IsCancellationRequested) + { + using (var wakeup = CancellationTokenSource.CreateLinkedTokenSource(_shutdownToken)) + { + wakeup.CancelAfter(TimeSpan.FromSeconds(1)); + try + { + await _subscriptionChanged.WaitAsync(wakeup.Token).AnyContext(); + } + catch (OperationCanceledException) when (!_shutdown.IsCancellationRequested) + { + // Periodically check transport health even without a local subscription change. + } + } + if (IsDisposed) + break; + if (_permanentSubscriberFault || _subscribers.IsEmpty) + { + using (await _subscriberLock.LockAsync(_shutdownToken).AnyContext()) + { + // Recheck under the lock: another caller may have registered while we waited. + if (_permanentSubscriberFault || _subscribers.IsEmpty) + { + await ClearSubscriberChannelAsync().AnyContext(); + if (_permanentSubscriberFault) + await ClearSubscriberConnectionAsync().AnyContext(); + } + } + continue; + } + if (_subscriberRecovering) + continue; + try + { + await EnsureTopicSubscriptionAsync(DisposedCancellationToken).AnyContext(); + } + catch (OperationCanceledException) when (_shutdown.IsCancellationRequested) + { + break; + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + _subscriptionError = exception; + _logger.LogError(exception, "Subscription repair failed; permanent fault: {Permanent}", _permanentSubscriberFault); + } + } + } + catch (OperationCanceledException) when (_shutdown.IsCancellationRequested) { } } - private Task OnSubscriberConnectionOnConnectionBlockedAsync(object sender, ConnectionBlockedEventArgs e) + private Task OnSubscriberConnectionOnCallbackExceptionAsync(object sender, CallbackExceptionEventArgs e) { - _logger.LogError("Subscriber connection blocked: {Reason}", e.Reason); + _logger.LogError(e.Exception, "Subscriber callback failed"); return Task.CompletedTask; } private Task OnSubscriberConnectionOnConnectionRecoveryErrorAsync(object sender, ConnectionRecoveryErrorEventArgs e) { - _logger.LogError(e.Exception, "Subscriber connection recovery error: {Message}", e.Exception.Message); + _subscriptionError = e.Exception; + _permanentSubscriberFault = IsPermanentSubscriptionError(e.Exception); + _logger.LogError(e.Exception, "Subscriber connection recovery failed"); return Task.CompletedTask; } private Task OnSubscriberConnectionOnConnectionShutdownAsync(object sender, ShutdownEventArgs e) { - _logger.LogInformation(e.Exception, "Subscriber shutdown. Reply Code: {ReplyCode} Reason: {ReplyText} Initiator: {Initiator}", e.ReplyCode, e.ReplyText, e.Initiator); + if (e.Initiator != ShutdownInitiator.Application && !IsDisposed) + _subscriberRecovering = true; + InvalidateDeliveries(); + _logger.LogInformation("Subscriber connection shutdown: {ReplyCode} {ReplyText}", e.ReplyCode, e.ReplyText); return Task.CompletedTask; } - private Task OnSubscriberConnectionOnConnectionUnblockedAsync(object sender, AsyncEventArgs e) + private Task OnSubscriberConnectionOnRecoverySucceededAsync(object sender, AsyncEventArgs e) { - _logger.LogInformation("Subscriber connection unblocked"); + _subscriptionError = null; + _subscriberRecovering = false; + _logger.LogInformation("Subscriber connection and topology recovery completed"); return Task.CompletedTask; } - private Task OnSubscriberConnectionOnRecoveringConsumerAsync(object sender, RecoveringConsumerEventArgs e) + private Task OnConsumerShutdownAsync(object sender, ShutdownEventArgs e) { - _logger.LogInformation("Subscriber connection recovering: {ConsumerTag}", e.ConsumerTag); + _logger.LogInformation("Consumer channel shutdown: {ReplyCode} {ReplyText}", e.ReplyCode, e.ReplyText); return Task.CompletedTask; } - private Task OnSubscriberConnectionOnRecoverySucceededAsync(object sender, AsyncEventArgs e) + private Task OnSubscriberChannelShutdownAsync(object sender, ShutdownEventArgs e) { - _logger.LogInformation("Subscriber connection recovery succeeded"); + InvalidateDeliveries(); return Task.CompletedTask; } - private Task OnConsumerShutdownAsync(object sender, ShutdownEventArgs e) + private Task OnConsumerUnregisteredAsync(object sender, ConsumerEventArgs e) { - _logger.LogInformation(e.Exception, "Consumer shutdown. Reply Code: {ReplyCode} Reason: {ReplyText} Initiator: {Initiator}", e.ReplyCode, e.ReplyText, e.Initiator); + InvalidateDeliveries(); return Task.CompletedTask; } - private async Task OnMessageAsync(object sender, BasicDeliverEventArgs envelope) + private Task OnQueueNameChangedAfterRecoveryAsync(object sender, QueueNameChangedAfterRecoveryEventArgs e) { - if (_subscriberChannel is not { } subscriberChannel) - { - _logger.LogDebug("Ignoring message because subscriber channel is not available ({MessageId})", - envelope.BasicProperties?.MessageId); - return; - } - - using var _ = _logger.BeginScope(s => s - .Property("MessageId", envelope.BasicProperties.MessageId) - .Property("DeliveryTag", envelope.DeliveryTag)); + if (String.Equals(_subscriptionQueueName, e.NameBefore, StringComparison.Ordinal)) + _subscriptionQueueName = e.NameAfter; + return Task.CompletedTask; + } - _logger.LogTrace("OnMessageAsync({MessageId})", envelope.BasicProperties.MessageId); + private static bool IsPermanentSubscriptionError(Exception exception) => + exception is OperationInterruptedException { ShutdownReason.ReplyCode: 403 or 406 or 530 } + || (exception.InnerException is not null && IsPermanentSubscriptionError(exception.InnerException)); - if (_subscribers.IsEmpty) - { - _logger.LogTrace("No subscribers ({MessageId})", envelope.BasicProperties.MessageId); - if (_options.AcknowledgementStrategy == AcknowledgementStrategy.Automatic) - await subscriberChannel.BasicRejectAsync(envelope.DeliveryTag, true).AnyContext(); + private void InvalidateDeliveries() + { + var previous = Interlocked.Exchange(ref _deliveryEpoch, new DeliveryEpoch(_shutdownToken)); + previous.Cancel(_logger); + } + private async Task OnMessageAsync(object sender, BasicDeliverEventArgs envelope) + { + if (IsDisposed || sender is not AsyncEventingBasicConsumer consumer || !ReferenceEquals(consumer, _consumer)) return; - } - + var channel = consumer.Channel; + var epoch = Volatile.Read(ref _deliveryEpoch); + string? queueName = _subscriptionQueueName; + using var lifetime = envelope.CancellationToken.CanBeCanceled + ? CancellationTokenSource.CreateLinkedTokenSource(epoch.Token, envelope.CancellationToken) : null; + var token = lifetime?.Token ?? epoch.Token; + using var scope = _deliveryScope(_logger, envelope.BasicProperties.MessageId, envelope.DeliveryTag); + Interlocked.Increment(ref _activeDeliveries); + bool blocked = false; try { - var message = ConvertToMessage(envelope); - await SendMessageToSubscribersAsync(message).AnyContext(); + long retryCount = -1; + while (true) + { + // Maintenance closes an unused consumer. Until then a quick resubscription + // may use this retained delivery; do not ACK an empty dispatch snapshot. + while (_subscribers.IsEmpty) + await Task.Delay(TimeSpan.FromMilliseconds(100), token).AnyContext(); + token.ThrowIfCancellationRequested(); + Exception? failure = null; + try + { + var message = ConvertToMessage(envelope); + if (retryCount >= 0) + message.Properties[RabbitMQConstants.XDeliveryCountHeader] = FormattableString.Invariant($"{retryCount}"); + if (_options.RequireSuccessfulDispatch) + { + while (!await DispatchRequiredAsync(message, token).AnyContext()) + await Task.Delay(TimeSpan.FromMilliseconds(100), token).AnyContext(); + } + else if (_options.AcknowledgementStrategy == AcknowledgementStrategy.Automatic) + { + // Shared dispatch passes the subscription token; flow this delivery's + // token through its scheduled callbacks without replacing the dispatcher. + var previous = _deliveryCancellation.Value; + _deliveryCancellation.Value = token; + try + { + await SendMessageToSubscribersAsync(message).WaitAsync(token).AnyContext(); + } + finally + { + _deliveryCancellation.Value = previous; + } + } + else + { + await SendMessageToSubscribersAsync(message).WaitAsync(token).AnyContext(); + } + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + return; + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + failure = exception; + } - if (_options.AcknowledgementStrategy == AcknowledgementStrategy.Automatic) - await subscriberChannel.BasicAckAsync(envelope.DeliveryTag, false).AnyContext(); + if (_options.AcknowledgementStrategy != AcknowledgementStrategy.Automatic) + { + if (failure is not null) + _logger.LogError(failure, "Best-effort handler failed after broker automatic acknowledgement"); + return; + } + if (!CanSettle(channel, epoch)) + return; + if (failure is null) + { + await channel.BasicAckAsync(envelope.DeliveryTag, false, token).AnyContext(); + _deliveryError = null; + return; + } + + if (retryCount < 0) + retryCount = RabbitMQMessageConverter.GetRetryCountFromHeader(envelope); + bool terminal = failure is InvalidDeliveryException || retryCount == Int64.MaxValue + || (_options.DeliveryLimit >= 0 && retryCount >= _options.DeliveryLimit); + if (terminal) + { + await CompleteTerminalDeliveryAsync(envelope, channel, epoch, failure, retryCount, token).AnyContext(); + } + else if (_topology.IsQuorumQueue) + { + if (CanSettle(channel, epoch)) + await channel.BasicRejectAsync(envelope.DeliveryTag, true, token).AnyContext(); + } + else + { + if (String.IsNullOrEmpty(queueName)) + { + await RetainDeliveryAsync(new MessageBusException("The actual subscription queue is unavailable for a local retry."), token).AnyContext(); + return; + } + var properties = RabbitMQMessageConverter.CopyHandoffProperties(envelope); + properties.Headers![RabbitMQConstants.XDeliveryCountHeader] = retryCount + 1; + try + { + await TransferAndAcknowledgeAsync(envelope, channel, epoch, String.Empty, queueName, properties, + retryInPlaceOnNack: true, token).AnyContext(); + } + catch (PublishException exception) when (!exception.IsReturn) + { + // The source queue can be full while this callback prevents it from draining. + // Keep the original unacknowledged and consume the same finite retry budget locally. + _deliveryError = exception; + if (!blocked) + { + Interlocked.Increment(ref _blockedDeliveries); + blocked = true; + } + retryCount++; + _logger.LogWarning(exception, "Retry publication was rejected; retrying the retained delivery locally"); + await Task.Delay(TimeSpan.FromSeconds(Math.Min(retryCount, 10)), token).AnyContext(); + continue; + } + } + return; + } } - catch (OperationCanceledException) + catch (OperationCanceledException) when (token.IsCancellationRequested) { - if (_options.AcknowledgementStrategy == AcknowledgementStrategy.Automatic) - await subscriberChannel.BasicRejectAsync(envelope.DeliveryTag, true).AnyContext(); + // Never acknowledge a delivery whose transport generation has ended. } - catch (MessageBusException) + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) { - // SendMessageToSubscribersAsync already logged the error - if (_options.AcknowledgementStrategy == AcknowledgementStrategy.Automatic) - await HandleDeliveryLimitsAsync(envelope).AnyContext(); + _deliveryError = exception; + _logger.LogError(exception, "Delivery settlement did not complete; no successful processing is asserted"); + if (CanSettle(channel, epoch)) + { + try + { + await RetainDeliveryAsync(exception, token).AnyContext(); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) { } + } } - catch (Exception ex) + finally { - _logger.LogError(ex, "Error handling message ({MessageId}): {Message}", envelope.BasicProperties.MessageId, ex.Message); - if (_options.AcknowledgementStrategy == AcknowledgementStrategy.Automatic) - await HandleDeliveryLimitsAsync(envelope).AnyContext(); + if (blocked) + Interlocked.Decrement(ref _blockedDeliveries); + Interlocked.Decrement(ref _activeDeliveries); } } - private async Task HandleDeliveryLimitsAsync(BasicDeliverEventArgs envelope) + private bool CanSettle(IChannel channel, DeliveryEpoch epoch) => + !IsDisposed && !epoch.Token.IsCancellationRequested && ReferenceEquals(epoch, Volatile.Read(ref _deliveryEpoch)) + && ReferenceEquals(channel, _subscriberChannel) && channel.IsOpen; + + private async Task CompleteTerminalDeliveryAsync(BasicDeliverEventArgs envelope, IChannel channel, + DeliveryEpoch epoch, Exception failure, long retryCount, CancellationToken cancellationToken) { - if (_subscriberChannel is not { } subscriberChannel) + if (!_options.RequireSuccessfulDispatch) { - _logger.LogWarning("Subscriber channel is not available; skipping delivery limit handling for message ({MessageId})", envelope.BasicProperties.MessageId); + // Let broker arguments or policies select the DLX; without one, RabbitMQ discards. + _logger.LogWarning("Delivery exhausted its retry budget; rejecting for broker dead-lettering or discard"); + if (CanSettle(channel, epoch)) + await channel.BasicRejectAsync(envelope.DeliveryTag, false, cancellationToken).AnyContext(); + _deliveryError = null; return; } - // Rule 1: If the limit is negative, reject regardless of queue type - if (_options.DeliveryLimit < 0) + var properties = RabbitMQMessageConverter.CopyHandoffProperties(envelope); + if (properties.Expiration is not null) + properties.Headers![RabbitMQConstants.XOriginalExpirationHeader] = properties.Expiration; + properties.Expiration = null; + properties.Headers![RabbitMQConstants.XDeliveryCountHeader] = retryCount; + properties.Headers[RabbitMQConstants.FailureTypeHeader] = failure.GetType().Name; + properties.Headers[RabbitMQConstants.OriginalExchangeHeader] = envelope.Exchange; + properties.Headers[RabbitMQConstants.OriginalRoutingKeyHeader] = envelope.RoutingKey; + await TransferAndAcknowledgeAsync(envelope, channel, epoch, _options.DeadLetterExchange!, + _options.DeadLetterRoutingKey ?? envelope.RoutingKey, properties, retryInPlaceOnNack: false, cancellationToken).AnyContext(); + } + + private async Task RetainDeliveryAsync(Exception exception, CancellationToken cancellationToken) + { + _deliveryError = exception; + Interlocked.Increment(ref _blockedDeliveries); + try { - _logger.LogDebug("Message ({MessageId}) rejected due to negative delivery limit ({DeliveryLimit})", - envelope.BasicProperties.MessageId, _options.DeliveryLimit); - await subscriberChannel.BasicRejectAsync(envelope.DeliveryTag, true).AnyContext(); - return; + _logger.LogError(exception, "Retaining delivery without acknowledgement; repair the destination/configuration before replay"); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).AnyContext(); } + finally + { + Interlocked.Decrement(ref _blockedDeliveries); + } + } - // Rule 2: Determine retry count from headers - long retryCount = GetRetryCountFromHeader(envelope); - - _logger.LogDebug("Processing message ({MessageId}) with delivery count {DeliveryCount} of {DeliveryLimit} (Queue type: {QueueType})", - envelope.BasicProperties.MessageId, retryCount, _options.DeliveryLimit, _isQuorumQueue ? "quorum" : "classic"); - - // Rule 3: Handle messages that have exceeded the delivery limit - if (retryCount >= _options.DeliveryLimit) + private async Task TransferAndAcknowledgeAsync(BasicDeliverEventArgs envelope, IChannel source, + DeliveryEpoch epoch, string exchange, string routingKey, BasicProperties properties, bool retryInPlaceOnNack, CancellationToken cancellationToken) + { + // A lost confirmation can produce duplicates. Keep the original until the + // replacement's confirmation AND routing are known; preserve logical identity. + var body = envelope.Body; + int attempts = 0; + bool blocked = false; + try { - if (_isQuorumQueue) + while (CanSettle(source, epoch)) { - // Check if we're significantly over the limit (suggests broker config mismatch) - if (retryCount >= _options.DeliveryLimit + 1) + cancellationToken.ThrowIfCancellationRequested(); + try { - _logger.LogWarning( - "Quorum queue message ({MessageId}) delivery count ({DeliveryCount}) is over configured limit ({DeliveryLimit})", - envelope.BasicProperties.MessageId, retryCount, _options.DeliveryLimit); - await subscriberChannel.BasicAckAsync(envelope.DeliveryTag, false).AnyContext(); + using var attempt = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + attempt.CancelAfter(TimeSpan.FromSeconds(10)); + await PublishHandoffAsync(exchange, routingKey, properties, body, attempt.Token).AnyContext(); } - else + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - _logger.LogDebug( - "Quorum queue message ({MessageId}) has exceeded delivery limit ({DeliveryLimit}): Rejecting to let broker handle", - envelope.BasicProperties.MessageId, _options.DeliveryLimit); - await subscriberChannel.BasicRejectAsync(envelope.DeliveryTag, true).AnyContext(); + return; } + catch (PublishException exception) when (retryInPlaceOnNack && !exception.IsReturn) + { + throw; + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + _deliveryError = exception; + if (!blocked) + { + Interlocked.Increment(ref _blockedDeliveries); + blocked = true; + } + attempts = Math.Min(attempts + 1, 10); + _logger.LogWarning(exception, "Retry/terminal handoff failed or is ambiguous; retaining original delivery (backoff {Seconds}s)", attempts); + await Task.Delay(TimeSpan.FromSeconds(attempts), cancellationToken).AnyContext(); + continue; + } + // ACK failure must not itself republish another replacement from this callback. + if (CanSettle(source, epoch)) + await source.BasicAckAsync(envelope.DeliveryTag, false, cancellationToken).AnyContext(); + _deliveryError = null; + return; } - else - { - _logger.LogDebug( - "Classic queue message ({MessageId}) has reached the delivery limit of {DeliveryLimit}: Acknowledging message", - envelope.BasicProperties.MessageId, _options.DeliveryLimit); - await subscriberChannel.BasicAckAsync(envelope.DeliveryTag, false).AnyContext(); - } - - return; } - - // Rule 4: Handle messages under the delivery limit - if (_isQuorumQueue) - { - _logger.LogDebug( - "Quorum queue message ({MessageId}) under delivery limit: Rejecting for broker-managed redelivery", - envelope.BasicProperties.MessageId); - await subscriberChannel.BasicRejectAsync(envelope.DeliveryTag, true).AnyContext(); - } - else + finally { - await RepublishMessageWithIncrementedDeliveryCountAsync(envelope, retryCount).AnyContext(); + if (blocked) + Interlocked.Decrement(ref _blockedDeliveries); } } - private async Task RepublishMessageWithIncrementedDeliveryCountAsync(BasicDeliverEventArgs envelope, long currentRetryCount) + /// + /// Publish a replacement on a dedicated confirmed channel with mandatory routing. + /// Completion means broker confirmation without a return, not downstream processing. + /// + protected virtual async Task PublishHandoffAsync(string exchange, string routingKey, BasicProperties properties, + ReadOnlyMemory body, CancellationToken cancellationToken) { - if (_subscriberChannel is not { } subscriberChannel) + using (await _handoffLock.LockAsync(cancellationToken).AnyContext()) { - _logger.LogWarning("Skipping republish for message ({MessageId}) because the subscriber channel is unavailable; leaving message unacknowledged for broker redelivery", - envelope.BasicProperties.MessageId); - return; + cancellationToken.ThrowIfCancellationRequested(); + try + { + if (_handoffConnection is not { IsOpen: true }) + { + await ClearHandoffTransportAsync().AnyContext(); + _handoffConnection = await CreateConnectionAsync(cancellationToken).AnyContext(); + } + if (_handoffChannel is not { IsOpen: true }) + { + await DisposeTransportAsync(_handoffChannel, "handoff channel").AnyContext(); + _handoffChannel = await _handoffConnection.CreateChannelAsync( + new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true), + cancellationToken).AnyContext(); + } + await _handoffChannel.BasicPublishAsync(exchange, routingKey, mandatory: true, properties, body, cancellationToken).AnyContext(); + } + catch (PublishException) when (_handoffConnection is { IsOpen: true } && _handoffChannel is { IsOpen: true }) + { + // A NACK or mandatory return rejects this publication, not the transport. + throw; + } + catch + { + await ClearHandoffTransportAsync().AnyContext(); + throw; + } } + } - string? originalMessageId = GetOriginalMessageIdFromHeader(envelope); - var properties = new BasicProperties(envelope.BasicProperties) - { - MessageId = Guid.NewGuid().ToString("N") - }; - - var headers = new Dictionary(envelope.BasicProperties.Headers ?? new Dictionary()) - { - [XDeliveryCountHeader] = currentRetryCount + 1, - [XOriginalMessageIdHeader] = originalMessageId - }; - - if (!String.IsNullOrEmpty(Activity.Current?.TraceStateString)) - headers["TraceState"] = Activity.Current.TraceStateString; - - properties.Headers = headers; - + protected override object? DeserializeMessageBody(IMessage message) + { + if (!_options.RequireSuccessfulDispatch) + return base.DeserializeMessageBody(message); try { - await EnsureTopicCreatedAsync(envelope.CancellationToken).AnyContext(); - await PublishMessageAsync(envelope.Exchange, envelope.RoutingKey, envelope.Body, properties, envelope.CancellationToken).AnyContext(); - await subscriberChannel.BasicAckAsync(envelope.DeliveryTag, false).AnyContext(); - - _logger.LogDebug("Republished classic queue message ({MessageId}) (OriginalMessageId={OriginalMessageId}) with delivery count {DeliveryCount}", - envelope.BasicProperties.MessageId, originalMessageId, currentRetryCount + 1); + var type = message.ClrType ?? GetMappedMessageType(message.Type) + ?? throw new InvalidDeliveryException("The required typed message schema could not be resolved."); + if (message.Data.IsEmpty) + throw new InvalidDeliveryException("The required typed message body is empty."); + return _serializer.Deserialize(message.Data, type) + ?? throw new InvalidDeliveryException("The required typed message body deserialized to null."); } - catch (Exception ex) + catch (InvalidDeliveryException) { - _logger.LogError(ex, "Failed to republish message ({MessageId}), acknowledging to prevent infinite retry", envelope.BasicProperties.MessageId); - await subscriberChannel.BasicAckAsync(envelope.DeliveryTag, false).AnyContext(); + throw; } - } - - /// - /// For quorum queues: x-delivery-count is set by broker (1 on first redelivery, absent on the first attempt) - /// For classic queues: x-delivery-count is only present if we added it during previous requeue - /// - private static long GetRetryCountFromHeader(BasicDeliverEventArgs envelope) - { - long retryCount = 0; - if (envelope.BasicProperties.Headers?.TryGetValue(XDeliveryCountHeader, out object? xDeliveryCount) is true) + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) { - if (!Int64.TryParse(xDeliveryCount?.ToString(), out retryCount)) - retryCount = 0; + throw new InvalidDeliveryException("The required typed message body could not be deserialized.", exception); } - - return retryCount; } - private static string? GetOriginalMessageIdFromHeader(BasicDeliverEventArgs envelope) + private async Task DispatchRequiredAsync(IMessage message, CancellationToken cancellationToken) { - if (envelope.BasicProperties.Headers?.TryGetValue(XOriginalMessageIdHeader, out object? xOriginalMessageId) is true) + cancellationToken.ThrowIfCancellationRequested(); + var subscribers = GetMessageSubscribers(message) + .Where(subscriber => !subscriber.CancellationToken.IsCancellationRequested).ToArray(); + if (subscribers.Length == 0) { - return xOriginalMessageId switch - { - string str => str, - byte[] bytes => Encoding.UTF8.GetString(bytes), - null => envelope.BasicProperties.MessageId, - _ => xOriginalMessageId.ToString() ?? envelope.BasicProperties.MessageId - }; + if (_subscribers.IsEmpty) + return false; + throw new InvalidDeliveryException("No matching live handler exists for a required delivery."); } + object? body = null; + if (subscribers.Any(subscriber => subscriber.Type != typeof(IMessage))) + body = message.GetBody() ?? throw new InvalidDeliveryException("The required typed payload is null."); - return envelope.BasicProperties.MessageId; + var handlers = subscribers.Select(subscriber => Task.Run(async () => + { + using var lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, subscriber.CancellationToken); + using var activity = StartHandleMessageActivity(message); + try + { + lifetime.Token.ThrowIfCancellationRequested(); + object value = subscriber.Type == typeof(IMessage) ? message + : subscriber.GenericType is not null + ? Activator.CreateInstance(subscriber.GenericType, message) + ?? throw new InvalidDeliveryException("A required typed message wrapper could not be created.") + : body!; + await subscriber.Action(value, lifetime.Token).WaitAsync(lifetime.Token).AnyContext(); + return !subscriber.CancellationToken.IsCancellationRequested; + } + catch (OperationCanceledException) when (subscriber.CancellationToken.IsCancellationRequested && !cancellationToken.IsCancellationRequested) + { + // A removed subscription must not prevent other live handlers from progressing. + return false; + } + catch (Exception exception) + { + activity?.SetErrorStatus(exception); + throw; + } + }, cancellationToken)).ToArray(); + var completed = await Task.WhenAll(handlers).WaitAsync(cancellationToken).AnyContext(); + cancellationToken.ThrowIfCancellationRequested(); + return completed.Any(value => value); } - private async Task PublishMessageAsync(string exchange, string routingKey, ReadOnlyMemory body, BasicProperties properties, CancellationToken cancellationToken) + private async Task PublishMessageAsync(string exchange, string routingKey, ReadOnlyMemory body, + BasicProperties properties, CancellationToken cancellationToken) { + using var lifetime = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownToken); + var token = lifetime.Token; await _resiliencePolicy.ExecuteAsync(async _ => { + token.ThrowIfCancellationRequested(); if (!_publisherReady.IsSet) { if (_options.PublishRecoveryTimeout <= TimeSpan.Zero) throw new MessageBusException("Cannot publish: publisher channel is closed or unavailable."); - - _logger.LogDebug("Publisher waiting for connection recovery..."); try { - await _publisherReady.WaitAsync(cancellationToken) - .WaitAsync(_options.PublishRecoveryTimeout, cancellationToken).AnyContext(); + await _publisherReady.WaitAsync(token).WaitAsync(_options.PublishRecoveryTimeout, token).AnyContext(); } catch (TimeoutException) { - throw new MessageBusException( - $"Publish failed: connection recovery did not complete within {_options.PublishRecoveryTimeout.TotalMilliseconds:F0}ms timeout."); + throw new MessageBusException(FormattableString.Invariant($"Publish failed: connection recovery did not complete within {_options.PublishRecoveryTimeout.TotalMilliseconds:F0}ms timeout.")); } } - - // PERF: Single lock serializes all publishes; with publisher confirms this limits throughput to 1 RTT. - // Consider channel pooling or batch publishing for high-throughput scenarios. - using (await _lock.LockAsync(cancellationToken).AnyContext()) + using (await _lock.LockAsync(token).AnyContext()) { + token.ThrowIfCancellationRequested(); if (_publisherChannel is not { IsOpen: true } channel) throw new MessageBusException("Cannot publish: publisher channel is closed or unavailable."); - - // Fail fast on broker resource alarms -- retrying would add pressure to a constrained broker. - // Unlike connection drops (which use the recovery gate to wait), blocked state has no recovery signal timing. if (_isPublisherBlocked) - throw new MessageBusException( - $"Cannot publish: publisher connection is blocked by broker ({_publisherBlockedReason ?? "resource alarm"})"); - - await channel.BasicPublishAsync(exchange, routingKey, mandatory: false, properties, body, cancellationToken: cancellationToken); + throw new MessageBusException($"Cannot publish: publisher connection is blocked by broker ({_publisherBlockedReason ?? "resource alarm"})"); + await channel.BasicPublishAsync(exchange, routingKey, _options.RequirePublishRouting, properties, body, token).AnyContext(); } - }, cancellationToken).AnyContext(); + }, token).AnyContext(); } - protected virtual IMessage ConvertToMessage(BasicDeliverEventArgs envelope) - { - // Zero-copy: envelope.Body is a pooled buffer that RabbitMQ.Client reclaims once OnMessageAsync - // returns. Referencing it without copying is safe because the body is deserialized synchronously - // within the awaited SendMessageToSubscribersAsync dispatch, before the handler returns. Per the - // IMessage.Data contract, the buffer is only valid for the duration of handling; consumers that - // retain the raw payload beyond the handler must copy it via ToArray(). - var message = new Message(envelope.Body, DeserializeMessageBody) - { - Type = envelope.BasicProperties.Type, - ClrType = GetMappedMessageType(envelope.BasicProperties.Type), - CorrelationId = envelope.BasicProperties.CorrelationId, - UniqueId = envelope.BasicProperties.MessageId - }; - - if (envelope.BasicProperties.Headers is not null) - foreach (var header in envelope.BasicProperties.Headers) - { - if (header.Value is byte[] byteData) - message.Properties[header.Key] = Encoding.UTF8.GetString(byteData); - else if (header.Value?.ToString() is { } stringValue) - message.Properties[header.Key] = stringValue; - } - - return message; - } + protected virtual IMessage ConvertToMessage(BasicDeliverEventArgs envelope) => + RabbitMQMessageConverter.Convert(envelope, GetMappedMessageType(envelope.BasicProperties.Type), _deserializeMessageBody); protected override async Task EnsureTopicCreatedAsync(CancellationToken cancellationToken) { - if (_publisherChannel is not null) + // A closed channel on a live connection needs repair. During network recovery, + // however, the client's recovery owns the existing channel and the publish gate. + if (_publisherChannel is { IsOpen: true } || (_publisherConnection is not null && !_publisherReady.IsSet)) return; - using (await _lock.LockAsync(cancellationToken).AnyContext()) + using var setup = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, _shutdownToken); + setup.CancelAfter(TimeSpan.FromSeconds(30)); + using (await _lock.LockAsync(setup.Token).AnyContext()) { - if (_publisherChannel is not null) + if (_publisherChannel is { IsOpen: true } || (_publisherConnection is not null && !_publisherReady.IsSet)) return; + setup.Token.ThrowIfCancellationRequested(); + if (IsDisposed) + throw new MessageBusException("Cannot initialize a disposed message bus."); - // Create the client connection, channel, declares the exchange, queue and binds - // the exchange with the publisher queue. It requires the name of our exchange, exchange type, durability and auto-delete. - // For now, we are using same autoDelete for both exchange and queue (it will survive a server restart) - _publisherConnection = await CreateConnectionAsync().AnyContext(); - DetectServerVersion(_publisherConnection); - RegisterPublisherConnectionEventHandlers(); - - // Reset blocked state after handlers are registered - new connections start unblocked - _isPublisherBlocked = false; - _publisherBlockedReason = null; - - _publisherChannel = await CreatePublisherChannelAsync(cancellationToken).AnyContext(); - - // We first attempt to create "x-delayed-type". For this the rabbitmq_delayed_message_exchange plugin should be installed. - // However, if the plugin is not installed this will throw an exception. In that case - // we attempt to create a regular exchange. If the regular exchange also throws an exception - // then troubleshoot the problem. - var delayedExchangeResult = await CreateDelayedExchangeAsync(_publisherChannel).AnyContext(); - if (delayedExchangeResult is null) + try { - await CreateRegularExchangeAsync(_publisherChannel).AnyContext(); + await DisposeTransportAsync(_publisherChannel, "publisher channel").AnyContext(); + _publisherChannel = null; + if (_publisherConnection is not { IsOpen: true }) + { + await ClearPublisherTransportAsync().AnyContext(); + _publisherConnection = await CreateConnectionAsync(setup.Token).AnyContext(); + RegisterPublisherConnectionEventHandlers(); + } + var serverVersion = _topology.DetectServerVersion(_publisherConnection); + _publisherChannel = await CreatePublisherChannelAsync(setup.Token).AnyContext(); + var delayed = await _topology.CreateDelayedExchangeAsync(_publisherChannel, serverVersion, setup.Token).AnyContext(); + if (delayed is false) + { + // An unknown exchange type may close the connection, not just the channel. + await ClearPublisherTransportAsync().AnyContext(); + _publisherConnection = await CreateConnectionAsync(setup.Token).AnyContext(); + RegisterPublisherConnectionEventHandlers(); + _publisherChannel = await CreatePublisherChannelAsync(setup.Token).AnyContext(); + } + if (delayed is not true) + await _topology.CreateRegularExchangeAsync(_publisherChannel, setup.Token).AnyContext(); + // Recreating a channel on an existing connection must not erase a broker alarm. + _publisherReady.Set(); } - else if (delayedExchangeResult is false) + catch { - // if the initial exchange creation was not successful, then we must close the previous connection - // and establish the new client connection and model; otherwise you will keep receiving failure in creation - // of the regular exchange too. - await _publisherChannel.DisposeAsync().AnyContext(); - UnregisterPublisherConnectionEventHandlers(); - await _publisherConnection.DisposeAsync().AnyContext(); - - _publisherConnection = await CreateConnectionAsync().AnyContext(); - DetectServerVersion(_publisherConnection); - RegisterPublisherConnectionEventHandlers(); - - // Reset blocked state after handlers are registered - _isPublisherBlocked = false; - _publisherBlockedReason = null; - - _publisherChannel = await CreatePublisherChannelAsync(cancellationToken).AnyContext(); - await CreateRegularExchangeAsync(_publisherChannel).AnyContext(); + await ClearPublisherTransportAsync().AnyContext(); + _publisherReady.Set(); + throw; } - - _logger.LogTrace("The unique channel number for the publisher is : {ChannelNumber}", _publisherChannel.ChannelNumber); } } private Task OnPublisherConnectionOnCallbackExceptionAsync(object sender, CallbackExceptionEventArgs e) { - _logger.LogError(e.Exception, "Publisher callback exception: {Message}", e.Exception.Message); + _logger.LogError(e.Exception, "Publisher callback exception"); return Task.CompletedTask; } @@ -547,30 +935,21 @@ private Task OnPublisherConnectionOnConnectionBlockedAsync(object sender, Connec { _publisherBlockedReason = e.Reason; _isPublisherBlocked = true; - _logger.LogError("Publisher connection blocked: {Reason}", e.Reason); + _logger.LogWarning("Publisher blocked by broker: {Reason}", e.Reason); return Task.CompletedTask; } private Task OnPublisherConnectionOnConnectionRecoveryErrorAsync(object sender, ConnectionRecoveryErrorEventArgs e) { - _logger.LogError(e.Exception, "Publisher connection recovery attempt failed, retrying: {Message}", e.Exception.Message); + _logger.LogError(e.Exception, "Publisher connection recovery failed"); return Task.CompletedTask; } private Task OnPublisherConnectionOnConnectionShutdownAsync(object sender, ShutdownEventArgs e) { if (e.Initiator != ShutdownInitiator.Application) - { _publisherReady.Reset(); - if (_options.PublishRecoveryTimeout > TimeSpan.Zero) - _logger.LogWarning("Publisher connection lost (Reply Code: {ReplyCode}, Reason: {ReplyText}). Publishes will wait up to {Timeout:g} for recovery.", - e.ReplyCode, e.ReplyText, _options.PublishRecoveryTimeout); - else - _logger.LogWarning("Publisher connection lost (Reply Code: {ReplyCode}, Reason: {ReplyText}). Publishes will fail immediately (recovery timeout disabled).", - e.ReplyCode, e.ReplyText); - } - - _logger.LogInformation(e.Exception, "Publisher shutdown. Reply Code: {ReplyCode} Reason: {ReplyText} Initiator: {Initiator}", e.ReplyCode, e.ReplyText, e.Initiator); + _logger.LogInformation("Publisher shutdown: {ReplyCode} {ReplyText}", e.ReplyCode, e.ReplyText); return Task.CompletedTask; } @@ -578,13 +957,6 @@ private Task OnPublisherConnectionOnConnectionUnblockedAsync(object sender, Asyn { _isPublisherBlocked = false; _publisherBlockedReason = null; - _logger.LogInformation("Publisher connection unblocked"); - return Task.CompletedTask; - } - - private Task OnPublisherConnectionOnRecoveringConsumerAsync(object sender, RecoveringConsumerEventArgs e) - { - _logger.LogInformation("Publisher connection recovering: {ConsumerTag}", e.ConsumerTag); return Task.CompletedTask; } @@ -593,433 +965,217 @@ private Task OnPublisherConnectionOnRecoverySucceededAsync(object sender, AsyncE _isPublisherBlocked = false; _publisherBlockedReason = null; _publisherReady.Set(); - _logger.LogInformation("Publisher connection recovery succeeded"); + _logger.LogInformation("Publisher connection recovered"); return Task.CompletedTask; } protected override async Task PublishImplAsync(string messageType, object message, MessageOptions options, CancellationToken cancellationToken) { byte[] data = SerializeMessageBody(messageType, message); - - // if the RabbitMQ plugin is not available, then use the base class delay mechanism - if (_delayedExchangePluginEnabled is false && options.DeliveryDelay.HasValue && options.DeliveryDelay.Value > TimeSpan.Zero) + bool delayed = options.DeliveryDelay.GetValueOrDefault() > TimeSpan.Zero; + if (delayed && _options.RequirePublishRouting) + throw new MessageBusException("Immediate routing confirmation is not supported for scheduled publications. Use a durable outbox or a separately verified scheduler."); + if (delayed && !_topology.SupportsDelayedExchange) { - _logger.LogTrace("Delayed message will be scheduled in-memory (broker-side delayed exchange unavailable): {MessageType} ({Delay}ms)", messageType, options.DeliveryDelay.Value.TotalMilliseconds); - var mappedType = GetMappedMessageType(messageType); - if (mappedType is null) - throw new MessageBusException($"Unable to resolve CLR type for delayed message: {messageType}"); - + if (_options.RequireBrokerDelayedDelivery) + throw new MessageBusException("Broker-side delayed delivery is required but unavailable; no in-memory message was scheduled."); + var mappedType = GetMappedMessageType(messageType) + ?? throw new MessageBusException($"Unable to resolve CLR type for delayed message: {messageType}"); + _logger.LogWarning("Scheduling a best-effort delayed message in process memory; it will not survive process termination ({MessageType})", messageType); SendDelayedMessage(mappedType, message, options); return; } - var basicProperties = new BasicProperties - { - MessageId = options.UniqueId ?? Guid.NewGuid().ToString("N"), - CorrelationId = options.CorrelationId, - Type = messageType - }; - - if (_options.IsDurable) - basicProperties.Persistent = true; - if (_options.DefaultMessageTimeToLive.HasValue) - basicProperties.Expiration = _options.DefaultMessageTimeToLive.Value.TotalMilliseconds.ToString(CultureInfo.InvariantCulture); - - if (options.Properties.TryGetValue(PriorityPropertyKey, out string? priorityValue) && Byte.TryParse(priorityValue, out byte priority)) - basicProperties.Priority = priority; - - if (options.Properties.Count > 0) - { - basicProperties.Headers ??= new Dictionary(); - foreach (var property in options.Properties.Where(p => !String.Equals(p.Key, PriorityPropertyKey, StringComparison.Ordinal))) - { - basicProperties.Headers.Add(property.Key, property.Value); - } - } - - // RabbitMQ only supports delayed messages with a third party plugin called "rabbitmq_delayed_message_exchange" - if (_delayedExchangePluginEnabled is true && options.DeliveryDelay.HasValue && options.DeliveryDelay.Value > TimeSpan.Zero) - { - // RabbitMQ's x-delay header must be a 32-bit signed int; the broker reads it as Int32 - // and negative values cause immediate delivery. - basicProperties.Headers ??= new Dictionary(); - double delayMs = options.DeliveryDelay.Value.TotalMilliseconds; - if (delayMs > Int32.MaxValue) - throw new ArgumentOutOfRangeException(nameof(options), $"DeliveryDelay ({options.DeliveryDelay.Value}) exceeds the maximum supported by RabbitMQ delayed exchange plugin ({Int32.MaxValue}ms)."); - basicProperties.Headers["x-delay"] = (int)delayMs; - _logger.LogTrace("Schedule delayed message: {MessageType} ({Delay}ms)", messageType, options.DeliveryDelay.Value.TotalMilliseconds); - } - else - { - _logger.LogTrace("Message publish type {MessageType} {MessageId}", messageType, basicProperties.MessageId); - } - - await PublishMessageAsync(_options.Topic, String.Empty, data, basicProperties, cancellationToken).AnyContext(); - _logger.LogDebug("Done publishing type {MessageType} {MessageId}", messageType, basicProperties.MessageId); + var properties = RabbitMQMessageConverter.CreateProperties(messageType, options, _options); + await PublishMessageAsync(_options.Topic, String.Empty, data, properties, cancellationToken).AnyContext(); } - private Task CreateConnectionAsync() - { - return _factory.CreateConnectionAsync(_endpoints); - } + private Task CreateConnectionAsync(CancellationToken cancellationToken = default) => + _factory.CreateConnectionAsync(_endpoints, cancellationToken: cancellationToken); private Task CreatePublisherChannelAsync(CancellationToken cancellationToken) { if (_publisherConnection is null) throw new MessageBusException("Publisher connection must be initialized before creating a channel."); - - if (_options.PublisherConfirmsEnabled) - { - var channelOptions = new CreateChannelOptions( - publisherConfirmationsEnabled: true, - publisherConfirmationTrackingEnabled: true); - return _publisherConnection.CreateChannelAsync(channelOptions, cancellationToken); - } - - return _publisherConnection.CreateChannelAsync(cancellationToken: cancellationToken); + var options = _options.PublisherConfirmsEnabled || _options.RequirePublishRouting + ? new CreateChannelOptions(publisherConfirmationsEnabled: true, publisherConfirmationTrackingEnabled: true) + : null; + return _publisherConnection.CreateChannelAsync(options, cancellationToken); } - private void DetectServerVersion(IConnection connection) - { - if (_serverVersion is not null) - return; - - var version = ParseServerVersion(connection.ServerProperties); - if (version is null) - return; + /// Parse the broker version from the UTF-8 AMQP server property. + public static Version? ParseServerVersion(IDictionary? serverProperties) => + RabbitMQTopology.ParseServerVersion(serverProperties); - _serverVersion = version; - _logger.LogDebug("Connected to RabbitMQ server version {ServerVersion}", version); - } - - /// - /// Parses the RabbitMQ broker version from AMQP server properties (the version field is UTF-8 bytes). - /// - public static Version? ParseServerVersion(IDictionary? serverProperties) + private async Task ClearPublisherTransportAsync() { - if (serverProperties?.TryGetValue("version", out var versionObj) is not true - || versionObj is not byte[] bytes) - return null; - - return Version.TryParse(Encoding.UTF8.GetString(bytes), out var version) ? version : null; + var channel = _publisherChannel; + _publisherChannel = null; + UnregisterPublisherConnectionEventHandlers(); + var connection = _publisherConnection; + _publisherConnection = null; + await DisposeTransportAsync(channel, "publisher channel").AnyContext(); + await DisposeTransportAsync(connection, "publisher connection").AnyContext(); + _isPublisherBlocked = false; + _publisherBlockedReason = null; } - /// - /// Attempts to create the delayed exchange. On RabbitMQ 4.3+ the probe is skipped because the - /// rabbitmq_delayed_message_exchange plugin depends on Mnesia which was removed. When the server - /// version could not be determined, the probe is still attempted so the plugin is used if available. - /// - /// - /// true if the delayed exchange was successfully declared (plugin is installed); - /// null if the probe was skipped or the result was already cached as disabled (the channel is still healthy); - /// false if the probe threw and the channel is likely closed. - /// - private async Task CreateDelayedExchangeAsync(IChannel channel) + private async Task ClearSubscriberChannelAsync() { - if (_delayedExchangePluginEnabled.HasValue) - return _delayedExchangePluginEnabled.Value ? true : null; - - if (_serverVersion is not null && _serverVersion >= _delayedExchangePluginIncompatibleVersion) - { - _logger.LogWarning( - "The rabbitmq_delayed_message_exchange plugin is incompatible with RabbitMQ {ServerVersion} (Mnesia removed). Delayed messages will be scheduled in-memory. Support for this plugin will be removed in a future version", - _serverVersion); - _delayedExchangePluginEnabled = false; - return null; - } - - bool success = true; - try - { - // This exchange is a delayed exchange (fanout). You need the rabbitmq_delayed_message_exchange plugin on RabbitMQ. - // Disclaimer: https://github.com/rabbitmq/rabbitmq-delayed-message-exchange/ - // Please read the *Performance Impact* of the delayed exchange type. - // On RabbitMQ 4.3+ this probe is skipped (see method summary); the plugin is incompatible. - var args = new Dictionary { { "x-delayed-type", ExchangeType.Fanout } }; - await channel.ExchangeDeclareAsync(_options.Topic, "x-delayed-message", _options.IsDurable, false, args).AnyContext(); - } - catch (OperationInterruptedException ex) - { - _logger.LogInformation( - ex, - "Unable to declare delayed exchange. ReplyCode: {ReplyCode}, ReplyText: {ReplyText}, Message: {Message}", - ex.ShutdownReason?.ReplyCode, - ex.ShutdownReason?.ReplyText, - ex.Message); - success = false; - } - - if (success) + if (_subscriberChannel is null && _consumer is null) + return; + InvalidateDeliveries(); + if (_consumer is not null) { - _logger.LogWarning( - "The rabbitmq_delayed_message_exchange plugin is deprecated and incompatible with RabbitMQ 4.3+. Support will be removed in a future version and delayed messages will fall back to in-memory scheduling. See https://github.com/rabbitmq/rabbitmq-delayed-message-exchange for details"); + _consumer.ReceivedAsync -= OnMessageAsync; + _consumer.ShutdownAsync -= OnConsumerShutdownAsync; + _consumer.UnregisteredAsync -= OnConsumerUnregisteredAsync; + _consumer = null; } - - _delayedExchangePluginEnabled = success; - return success; + var channel = _subscriberChannel; + _subscriberChannel = null; + if (channel is not null) + channel.ChannelShutdownAsync -= OnSubscriberChannelShutdownAsync; + await DisposeTransportAsync(channel, "subscriber channel").AnyContext(); } - private Task CreateRegularExchangeAsync(IChannel channel) + private async Task ClearSubscriberConnectionAsync() { - return channel.ExchangeDeclareAsync(_options.Topic, ExchangeType.Fanout, _options.IsDurable, false); + UnregisterSubscriberConnectionEventHandlers(); + var connection = _subscriberConnection; + _subscriberConnection = null; + _subscriberRecovering = false; + await DisposeTransportAsync(connection, "subscriber connection").AnyContext(); } - /// - /// The client sends a message to an exchange and attaches a routing key to it. - /// The message is sent to all queues with the matching routing key. Each queue has a - /// receiver attached which will process the message. We’ll initiate a dedicated message - /// exchange and not use the default one. Note that a queue can be dedicated to one or more routing keys. - /// - /// channel - private async Task CreateQueueAsync(IChannel channel) + private async Task ClearHandoffTransportAsync() { - // Set up the queue where the messages will reside - it requires the queue name and durability. - // Durable (the queue will survive a broker restart) - // Arguments (some brokers use it to implement additional features like message TTL) - var arguments = _options.Arguments is not null - ? new Dictionary(_options.Arguments) - : new Dictionary(); - - if (!String.IsNullOrWhiteSpace(_options.DeadLetterExchange)) - { - arguments["x-dead-letter-exchange"] = _options.DeadLetterExchange; - - if (!String.IsNullOrWhiteSpace(_options.DeadLetterRoutingKey)) - arguments["x-dead-letter-routing-key"] = _options.DeadLetterRoutingKey; - - if (_options.DeadLetterStrategy.HasValue) - { - if (_options.DeadLetterStrategy == DeadLetterStrategy.AtLeastOnce) - { - if (!_isQuorumQueue) - throw new MessageBusException("At-least-once dead-lettering requires quorum queues. Call UseQuorumQueues()."); - - if (_options.Overflow != QueueOverflowBehavior.RejectPublish) - throw new MessageBusException("At-least-once dead-lettering requires overflow to be set to RejectPublish. Call .OverflowBehavior(QueueOverflowBehavior.RejectPublish)."); - } - - arguments["x-dead-letter-strategy"] = _options.DeadLetterStrategy.Value.ToEnumString(); - } - } - - if (_options.Overflow.HasValue) - arguments["x-overflow"] = _options.Overflow.Value.ToEnumString(); - - if (_options.ConsumerTimeout.HasValue) - { - if (!_isQuorumQueue) - throw new MessageBusException("Per-queue consumer timeout (x-consumer-timeout) requires quorum queues (RabbitMQ 4.3+). Call UseQuorumQueues() before ConsumerTimeout()."); - - if (_serverVersion is not null && _serverVersion < _delayedExchangePluginIncompatibleVersion) - throw new MessageBusException($"Per-queue consumer timeout (x-consumer-timeout) requires RabbitMQ 4.3+. Detected server version: {_serverVersion}."); - - arguments["x-consumer-timeout"] = (long)_options.ConsumerTimeout.Value.TotalMilliseconds; - } - - if (_options.SingleActiveConsumer) - arguments["x-single-active-consumer"] = true; - - if (_options.MaxPriority.HasValue) - arguments["x-max-priority"] = (int)_options.MaxPriority.Value; - - if (_options.DelayedRetryType.HasValue) - { - if (!_isQuorumQueue) - throw new MessageBusException("Delayed retries (x-delayed-retry-*) require quorum queues (RabbitMQ 4.3+). Call UseQuorumQueues() before UseDelayedRetries()."); - - if (_serverVersion is not null && _serverVersion < _delayedExchangePluginIncompatibleVersion) - throw new MessageBusException($"Delayed retries (x-delayed-retry-*) require RabbitMQ 4.3+. Detected server version: {_serverVersion}."); - - arguments["x-delayed-retry-type"] = _options.DelayedRetryType.Value.ToEnumString(); - if (_options.DelayedRetryMin.HasValue) - arguments["x-delayed-retry-min"] = _options.DelayedRetryMin.Value; - if (_options.DelayedRetryMax.HasValue) - arguments["x-delayed-retry-max"] = _options.DelayedRetryMax.Value; - } - - var result = await channel.QueueDeclareAsync(_options.SubscriptionQueueName, _options.IsDurable, _options.IsSubscriptionQueueExclusive, _options.SubscriptionQueueAutoDelete, arguments.Count > 0 ? arguments : null).AnyContext(); - string queueName = result.QueueName; - - // Bind the queue with the exchange. - await channel.QueueBindAsync(queueName, _options.Topic, String.Empty).AnyContext(); - - return queueName; + var channel = _handoffChannel; + var connection = _handoffConnection; + _handoffChannel = null; + _handoffConnection = null; + await DisposeTransportAsync(channel, "handoff channel").AnyContext(); + await DisposeTransportAsync(connection, "handoff connection").AnyContext(); } - private async Task ClosePublisherConnectionAsync() + private async Task DisposeTransportAsync(IAsyncDisposable? resource, string role) { - if (_publisherConnection is null) + if (resource is null) return; - - using (await _lock.LockAsync().AnyContext()) + try { - _logger.LogTrace("ClosePublisherConnectionAsync"); - - if (_publisherChannel is not null) - { - await _publisherChannel.DisposeAsync().AnyContext(); - _publisherChannel = null; - } - - if (_publisherConnection is not null) - { - UnregisterPublisherConnectionEventHandlers(); - await _publisherConnection.DisposeAsync().AnyContext(); - _publisherConnection = null; - } + await resource.DisposeAsync().AsTask().WaitAsync(_options.ShutdownTimeout).AnyContext(); } - } - - private async Task CloseSubscriberConnectionAsync(CancellationToken cancellationToken = default) - { - if (_subscriberConnection is null) - return; - - using (await _lock.LockAsync(cancellationToken).AnyContext()) + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) { - _logger.LogTrace("CloseSubscriberConnectionAsync"); - - if (_consumer is not null) - { - UnregisterConsumerEventHandlers(); - _consumer = null; - } - - if (_subscriberChannel is not null) - { - await _subscriberChannel.DisposeAsync().AnyContext(); - _subscriberChannel = null; - } - - if (_subscriberConnection is not null) - { - UnregisterSubscriberConnectionEventHandlers(); - await _subscriberConnection.DisposeAsync().AnyContext(); - _subscriberConnection = null; - } + _logger.LogWarning(exception, "Transport cleanup failed or exceeded its timeout ({Role})", role); } } - private static string SanitizeUri(Uri uri) - { - if (String.IsNullOrEmpty(uri.UserInfo)) - return uri.ToString(); - - string portSuffix = uri.IsDefaultPort ? "" : $":{uri.Port}"; - return $"{uri.Scheme}://***@{uri.Host}{portSuffix}{uri.AbsolutePath}"; - } - private void RegisterPublisherConnectionEventHandlers() { if (_publisherConnection is null) - throw new MessageBusException("Publisher connection must be initialized before registering event handlers."); - + throw new MessageBusException("Publisher connection has not been initialized."); _publisherConnection.CallbackExceptionAsync += OnPublisherConnectionOnCallbackExceptionAsync; _publisherConnection.ConnectionBlockedAsync += OnPublisherConnectionOnConnectionBlockedAsync; _publisherConnection.ConnectionRecoveryErrorAsync += OnPublisherConnectionOnConnectionRecoveryErrorAsync; _publisherConnection.ConnectionShutdownAsync += OnPublisherConnectionOnConnectionShutdownAsync; _publisherConnection.ConnectionUnblockedAsync += OnPublisherConnectionOnConnectionUnblockedAsync; - _publisherConnection.RecoveringConsumerAsync += OnPublisherConnectionOnRecoveringConsumerAsync; _publisherConnection.RecoverySucceededAsync += OnPublisherConnectionOnRecoverySucceededAsync; } private void UnregisterPublisherConnectionEventHandlers() { if (_publisherConnection is null) - throw new MessageBusException("Publisher connection must be initialized before unregistering event handlers."); - + return; _publisherConnection.CallbackExceptionAsync -= OnPublisherConnectionOnCallbackExceptionAsync; _publisherConnection.ConnectionBlockedAsync -= OnPublisherConnectionOnConnectionBlockedAsync; _publisherConnection.ConnectionRecoveryErrorAsync -= OnPublisherConnectionOnConnectionRecoveryErrorAsync; _publisherConnection.ConnectionShutdownAsync -= OnPublisherConnectionOnConnectionShutdownAsync; _publisherConnection.ConnectionUnblockedAsync -= OnPublisherConnectionOnConnectionUnblockedAsync; - _publisherConnection.RecoveringConsumerAsync -= OnPublisherConnectionOnRecoveringConsumerAsync; _publisherConnection.RecoverySucceededAsync -= OnPublisherConnectionOnRecoverySucceededAsync; } private void RegisterSubscriberConnectionEventHandlers() { if (_subscriberConnection is null) - throw new MessageBusException("Subscriber connection must be initialized before registering event handlers."); - - _subscriberConnection.CallbackExceptionAsync += OnSubscriberConnectionOnCallbackExceptionAsync; - _subscriberConnection.ConnectionBlockedAsync += OnSubscriberConnectionOnConnectionBlockedAsync; - _subscriberConnection.ConnectionRecoveryErrorAsync += OnSubscriberConnectionOnConnectionRecoveryErrorAsync; + throw new MessageBusException("Subscriber connection has not been initialized."); _subscriberConnection.ConnectionShutdownAsync += OnSubscriberConnectionOnConnectionShutdownAsync; - _subscriberConnection.ConnectionUnblockedAsync += OnSubscriberConnectionOnConnectionUnblockedAsync; - _subscriberConnection.RecoveringConsumerAsync += OnSubscriberConnectionOnRecoveringConsumerAsync; _subscriberConnection.RecoverySucceededAsync += OnSubscriberConnectionOnRecoverySucceededAsync; + _subscriberConnection.ConnectionRecoveryErrorAsync += OnSubscriberConnectionOnConnectionRecoveryErrorAsync; + _subscriberConnection.CallbackExceptionAsync += OnSubscriberConnectionOnCallbackExceptionAsync; + _subscriberConnection.QueueNameChangedAfterRecoveryAsync += OnQueueNameChangedAfterRecoveryAsync; } private void UnregisterSubscriberConnectionEventHandlers() { if (_subscriberConnection is null) - throw new MessageBusException("Subscriber connection must be initialized before unregistering event handlers."); - - _subscriberConnection.CallbackExceptionAsync -= OnSubscriberConnectionOnCallbackExceptionAsync; - _subscriberConnection.ConnectionBlockedAsync -= OnSubscriberConnectionOnConnectionBlockedAsync; - _subscriberConnection.ConnectionRecoveryErrorAsync -= OnSubscriberConnectionOnConnectionRecoveryErrorAsync; + return; _subscriberConnection.ConnectionShutdownAsync -= OnSubscriberConnectionOnConnectionShutdownAsync; - _subscriberConnection.ConnectionUnblockedAsync -= OnSubscriberConnectionOnConnectionUnblockedAsync; - _subscriberConnection.RecoveringConsumerAsync -= OnSubscriberConnectionOnRecoveringConsumerAsync; _subscriberConnection.RecoverySucceededAsync -= OnSubscriberConnectionOnRecoverySucceededAsync; + _subscriberConnection.ConnectionRecoveryErrorAsync -= OnSubscriberConnectionOnConnectionRecoveryErrorAsync; + _subscriberConnection.CallbackExceptionAsync -= OnSubscriberConnectionOnCallbackExceptionAsync; + _subscriberConnection.QueueNameChangedAfterRecoveryAsync -= OnQueueNameChangedAfterRecoveryAsync; } - private void RegisterConsumerEventHandlers() - { - if (_consumer is null) - throw new MessageBusException("Consumer must be initialized before registering event handlers."); + /// Close the recovery gate for deterministic gate tests. + internal void SimulatePublisherConnectionLost() => _publisherReady.Reset(); - _consumer.ReceivedAsync += OnMessageAsync; - _consumer.ShutdownAsync += OnConsumerShutdownAsync; - } + /// Open the recovery gate for deterministic gate tests. + internal void SimulatePublisherRecoverySucceeded() => _publisherReady.Set(); - private void UnregisterConsumerEventHandlers() + /// Simulate network loss, not a channel-only failure. + internal async Task SimulatePublisherConnectionShutdownAsync() { - if (_consumer is null) - throw new MessageBusException("Consumer must be initialized before unregistering event handlers."); - - _consumer.ReceivedAsync -= OnMessageAsync; - _consumer.ShutdownAsync -= OnConsumerShutdownAsync; + await OnPublisherConnectionOnConnectionShutdownAsync(this, + new ShutdownEventArgs(ShutdownInitiator.Library, 541, "Simulated connection reset")).AnyContext(); + _publisherChannel = null; } - /// - /// Simulates an unexpected publisher connection loss for testing the recovery gate. - /// Closes the gate so that subsequent publishes will wait for recovery. - /// - internal void SimulatePublisherConnectionLost() - { - _publisherReady.Reset(); - } + /// Verify a failed recovery attempt does not open the gate. + internal Task SimulatePublisherConnectionRecoveryErrorAsync() => + OnPublisherConnectionOnConnectionRecoveryErrorAsync(this, + new ConnectionRecoveryErrorEventArgs(new Exception("Simulated recovery failure"))); - /// - /// Simulates a successful publisher connection recovery for testing. - /// Opens the gate so that waiting publishes resume. - /// - internal void SimulatePublisherRecoverySucceeded() + private sealed class DeliveryEpoch { - _publisherReady.Set(); - } + private readonly CancellationTokenSource _source; + private int _cancelled; + internal CancellationToken Token { get; } + internal DeliveryEpoch(CancellationToken shutdown) + { + _source = CancellationTokenSource.CreateLinkedTokenSource(shutdown); + Token = _source.Token; + } - /// - /// Simulates a publisher connection shutdown event for testing. - /// Triggers the full shutdown handler (gate closure) and nulls the publisher channel - /// to represent a real unexpected disconnect. - /// - internal async Task SimulatePublisherConnectionShutdownAsync() - { - await OnPublisherConnectionOnConnectionShutdownAsync(this, - new ShutdownEventArgs(ShutdownInitiator.Library, 541, "Simulated connection reset")).AnyContext(); - _publisherChannel = null; + internal void Cancel(ILogger logger) + { + if (Interlocked.Exchange(ref _cancelled, 1) == 0) + _ = CancelAndDisposeAsync(logger); + } + + private async Task CancelAndDisposeAsync(ILogger logger) + { + try + { + // CancelAsync marks the token immediately but invokes callbacks off the + // transport event path. A slow handler cancellation cannot block recovery. + await _source.CancelAsync().AnyContext(); + } + catch (Exception exception) when (exception is not OutOfMemoryException and not StackOverflowException) + { + logger.LogWarning(exception, "A delivery cancellation callback failed"); + } + finally + { + _source.Dispose(); + } + } } - /// - /// Simulates a connection recovery error event for testing. - /// Verifies the gate remains closed (recovery continues retrying). - /// - internal Task SimulatePublisherConnectionRecoveryErrorAsync() + private sealed class InvalidDeliveryException : MessageBusException { - return OnPublisherConnectionOnConnectionRecoveryErrorAsync(this, - new ConnectionRecoveryErrorEventArgs(new Exception("Simulated recovery failure"))); + internal InvalidDeliveryException(string message) : base(message) { } + internal InvalidDeliveryException(string message, Exception inner) : base(message, inner) { } } } diff --git a/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBusOptions.cs b/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBusOptions.cs index 69c96dae..43967257 100644 --- a/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBusOptions.cs +++ b/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageBusOptions.cs @@ -1,20 +1,23 @@ using System; using System.Collections.Generic; +using Foundatio.Utility; namespace Foundatio.Messaging; public class RabbitMQMessageBusOptions : SharedMessageBusOptions { /// - /// The connection string. See https://www.rabbitmq.com/uri-spec.html for more information. - /// Provides credentials and vhost. When Hosts is specified, the host in the connection string is ignored. + /// The connection URI. Provides credentials and vhost. A nonempty Hosts list replaces the URI host. + /// The amqps scheme enables TLS for every endpoint; each certificate identity must match its endpoint. /// public string? ConnectionString { get; set; } /// - /// List of hosts for failover. When specified, the client will try each host in order until one succeeds. - /// Format: "hostname" or "hostname:port" (default port is 5672, or 5671 for amqps). - /// If not specified, the host from ConnectionString is used. + /// Replacement endpoints for connection establishment and network recovery, in hostname, + /// hostname:port, bare IPv6, or [IPv6]:port form. Selection is randomized, not ordered. + /// An omitted host-list port uses 5672 for amqp or 5671 for amqps, not a custom URI port. + /// Explicit ports must be 1-65535. A null or empty list uses the URI endpoint. + /// A nonempty list containing no usable endpoint is rejected. /// public IList? Hosts { get; set; } @@ -55,36 +58,38 @@ public class RabbitMQMessageBusOptions : SharedMessageBusOptions public string SubscriptionQueueName { get; set; } = String.Empty; /// - /// How messages should be acknowledged. + /// The default is FireAndForget (broker automatic acknowledgement), not acknowledgement + /// after successful application processing. Automatic acknowledges after dispatch succeeds. /// public AcknowledgementStrategy AcknowledgementStrategy { get; set; } = AcknowledgementStrategy.FireAndForget; /// - /// Consumer prefetch count. Limits the number of unacknowledged messages a consumer can have. - /// Set to 0 (default) which will prefetch all available messages (unbounded). - /// Recommended values: 5-50 for most scenarios, higher for fast processing, lower for slow/heavy processing or large message sizes. + /// Limits unacknowledged deliveries per consumer, not handler concurrency. + /// Does not bound FireAndForget consumers. When both prefetch settings are zero, + /// no BasicQos is sent and a broker-configured default can still apply. /// public ushort PrefetchCount { get; set; } - /// - /// Consumer prefetch size in bytes. Limits the total size of unacknowledged messages a consumer can have. - /// Set to 0 (default) for no size limit. This provides additional flow control beyond message count. - /// + /// Prefetch size in bytes. Zero is the supported RabbitMQ default; not an application memory limit. public uint PrefetchSize { get; set; } /// - /// Whether QoS settings apply globally to the connection or just to the channel. - /// When true, the QoS settings apply to all consumers on the connection. - /// When false (default), the QoS settings apply only to consumers on this channel. + /// Whether prefetch is shared by consumers on a channel, not across the connection. + /// Unsupported for quorum queues. The provider falls back to per-consumer QoS where necessary. /// - [Obsolete("Global QoS is deprecated in RabbitMQ 4.3+ and will be removed in a future version. Use per-channel prefetch (GlobalQos = false) instead.")] + [Obsolete("Global QoS is deprecated in RabbitMQ 4.3+ and will be removed in a future version. Use per-consumer prefetch (GlobalQos = false) instead.")] public bool GlobalQos { get; set; } /// - /// Sets the maximum number of times a message can be delivered (retried) before it is discarded. If using - /// classic queues, we will republish the message with an x-delivery-count header and acknowledge the previous message. - /// - /// Setting this to -1 means there is no limit on the number of deliveries. + /// Maximum failed redeliveries after the initial attempt; -1 means unlimited. + /// Classic retries use a confirmed, subscription-local handoff, requiring write permission on amq.default. + /// A broker NACK retries the retained delivery locally to avoid stalling a full source queue. + /// Local attempts count toward this limit but cannot persist across connection loss; unlimited retries + /// can block later deliveries. On exhaustion, the broker + /// dead-letters the delivery according to queue arguments or policies, or discards it without a DLX. + /// RequireSuccessfulDispatch instead uses a confirmed application terminal transfer. + /// Broker limits and policies are separate: + /// a finite quorum broker delivery limit can also act on connection-loss redeliveries. /// public long DeliveryLimit { get; set; } = 2; @@ -94,9 +99,40 @@ public class RabbitMQMessageBusOptions : SharedMessageBusOptions /// Performance impact varies by workload - use async/pipelining patterns for best results. /// Default: false (fire-and-forget publishing for backward compatibility). /// See: https://www.rabbitmq.com/docs/confirms#publisher-confirms + /// A confirmation does not guarantee consumer processing or routing to every subscription. /// public bool PublisherConfirmsEnabled { get; set; } + /// + /// Require at least one route for an ordinary immediate publication. Enables confirms and + /// AMQP mandatory returns. Does not verify all expected fanout subscriptions or consumer processing. + /// Scheduled publication through the delayed-exchange plugin cannot satisfy this contract. + /// + public bool RequirePublishRouting { get; set; } + + /// + /// Require actual completion of matching live handlers before acknowledgement. Requires Automatic, + /// a configured DeadLetterExchange, and confirmed terminal transfers. Provision the destination separately. + /// Failed terminal transfers retain the original delivery and retry until cancellation or repair. + /// Unmatched types and malformed typed payloads become terminal failures instead of intentional + /// pub/sub filtering. False preserves the existing best-effort filtering/dispatch contract. + /// + public bool RequireSuccessfulDispatch { get; set; } + + /// + /// Reject delayed publication when broker-side delayed delivery is unavailable; never silently + /// schedule that work in process memory. Requires durable publications and publisher confirms. + /// This is not a guarantee of replicated scheduling or eventual destination availability. + /// + public bool RequireBrokerDelayedDelivery { get; set; } + + /// + /// Maximum individual transport cleanup wait, including lock acquisition. Cleanup can continue after + /// the wait expires; this is not a total application shutdown deadline. Handlers receive cancellation; handlers that + /// ignore cancellation may continue running and must tolerate redelivery. Default: ten seconds. + /// + public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(10); + /// /// Maximum time each publish attempt will wait for connection recovery before failing. /// During a transient connection drop, publishes suspend (rather than failing immediately) @@ -111,7 +147,7 @@ public class RabbitMQMessageBusOptions : SharedMessageBusOptions /// /// Heartbeat timeout negotiated with the broker. Controls how quickly dead TCP connections are detected. /// Lower values detect failures faster but may cause false positives on congested networks. - /// Set to TimeSpan.Zero to disable heartbeats (not recommended for production). + /// TimeSpan.Zero disables heartbeats only when the broker also offers zero. /// Default: null (uses client library default of 60 seconds). /// See: https://www.rabbitmq.com/docs/heartbeats /// @@ -126,10 +162,11 @@ public class RabbitMQMessageBusOptions : SharedMessageBusOptions public TimeSpan? NetworkRecoveryInterval { get; set; } /// - /// Dead letter exchange name. Messages that exceed the delivery limit or are rejected - /// will be routed to this exchange instead of being dropped. - /// Set via the x-dead-letter-exchange queue argument. - /// See: https://www.rabbitmq.com/docs/dlx + /// Sets the broker x-dead-letter-exchange argument. Ordinary exhausted deliveries use broker + /// dead-lettering, which is at-most-once unless quorum at-least-once dead-lettering is configured. + /// RequireSuccessfulDispatch also uses this destination for confirmed, mandatory application transfers. + /// Provision the destination separately. Broker policies can configure a DLX without this option + /// for ordinary dispatch; required dispatch needs an explicit application transfer destination. /// public string? DeadLetterExchange { get; set; } @@ -141,11 +178,9 @@ public class RabbitMQMessageBusOptions : SharedMessageBusOptions public string? DeadLetterRoutingKey { get; set; } /// - /// Dead-letter strategy for quorum queues. Controls whether messages are transferred - /// to the DLX with at-most-once (default, may lose messages) or at-least-once (guaranteed delivery) - /// semantics. At-least-once requires Overflow to be set to RejectPublish. - /// Set via the x-dead-letter-strategy queue argument. - /// See: https://www.rabbitmq.com/docs/quorum-queues#dead-lettering + /// Broker dead-letter strategy for quorum queues. AtLeastOnce requires RejectPublish overflow, + /// a configured DLX, and the broker prerequisites. It retains transfers until confirmed; + /// duplicate transfers remain possible. AtMostOnce can lose messages in transit. /// public DeadLetterStrategy? DeadLetterStrategy { get; set; } @@ -168,11 +203,8 @@ public class RabbitMQMessageBusOptions : SharedMessageBusOptions public TimeSpan? ConsumerTimeout { get; set; } /// - /// When true, only one consumer at a time will receive messages from the queue. - /// Other consumers act as standby and automatically take over if the active consumer disconnects. - /// Useful for strict message ordering with automatic failover. - /// Set via the x-single-active-consumer queue argument. - /// See: https://www.rabbitmq.com/docs/consumers#single-active-consumer + /// Enable one active consumer with standby consumers. Does not eliminate duplicates, + /// guarantee business execution order, or override message priorities. /// public bool SingleActiveConsumer { get; set; } @@ -183,6 +215,7 @@ public class RabbitMQMessageBusOptions : SharedMessageBusOptions /// Set via the x-max-priority queue argument for classic queues only. /// RabbitMQ 4.2 quorum queues use normal/high tiers; RabbitMQ 4.3+ quorum queues support /// 32 strict priority levels automatically and cannot use this setting. + /// RabbitMQ.Client 7.2.2 omits a message priority of zero. /// See: https://www.rabbitmq.com/docs/priority /// See: https://www.rabbitmq.com/docs/4.2/priority#quorum-queues /// @@ -205,7 +238,7 @@ public byte? MaxPriority /// internal static bool IsQuorumQueue(IDictionary? arguments) { - return arguments is not null && arguments.TryGetValue("x-queue-type", out object? queueType) + return arguments is not null && arguments.TryGetValue(RabbitMQConstants.QueueTypeArgument, out object? queueType) && queueType is string type && String.Equals(type, "quorum", StringComparison.OrdinalIgnoreCase); } @@ -234,6 +267,27 @@ internal static bool IsQuorumQueue(IDictionary? arguments) /// Set via x-delayed-retry-max queue argument. /// public int? DelayedRetryMax { get; set; } + + internal static void Validate(RabbitMQMessageBusOptions options) + { + ArgumentOutOfRangeException.ThrowIfLessThan(options.DeliveryLimit, -1L); + ArgumentOutOfRangeException.ThrowIfLessThan(options.PublishRecoveryTimeout, TimeSpan.Zero); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(options.ShutdownTimeout, TimeSpan.Zero); + if (options.RequestedHeartbeat.HasValue) + ArgumentOutOfRangeException.ThrowIfLessThan(options.RequestedHeartbeat.Value, TimeSpan.Zero); + if (options.NetworkRecoveryInterval.HasValue) + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(options.NetworkRecoveryInterval.Value, TimeSpan.Zero); + if (options.MaxPriority.HasValue && IsQuorumQueue(options.Arguments)) + throw new InvalidOperationException("MaxPriority applies only to classic queues and cannot be used with quorum queues."); + if (options.RequireSuccessfulDispatch && options.AcknowledgementStrategy != AcknowledgementStrategy.Automatic) + throw new ArgumentException("RequireSuccessfulDispatch requires Automatic acknowledgements.", nameof(options)); + if (options.RequireSuccessfulDispatch && String.IsNullOrWhiteSpace(options.DeadLetterExchange)) + throw new ArgumentException("RequireSuccessfulDispatch requires a configured DeadLetterExchange for terminal deliveries.", nameof(options)); + if (options.RequireBrokerDelayedDelivery && (!options.IsDurable || (!options.PublisherConfirmsEnabled && !options.RequirePublishRouting))) + throw new ArgumentException("Required broker-side delay needs durable publications and publisher confirms.", nameof(options)); + if (!String.IsNullOrWhiteSpace(options.DeadLetterExchange) && String.Equals(options.DeadLetterExchange, options.Topic, StringComparison.Ordinal)) + throw new ArgumentException("The terminal exchange must differ from the source fanout exchange.", nameof(options)); + } } public class RabbitMQMessageBusOptionsBuilder : SharedMessageBusOptionsBuilder @@ -244,20 +298,14 @@ public RabbitMQMessageBusOptionsBuilder ConnectionString(string? connectionStrin return this; } - /// - /// Sets the list of hosts for failover support. Replaces the host from ConnectionString. - /// - /// Hostnames in format "hostname" or "hostname:port" + /// Replace the URI host with randomly selected hostnames or IPs, optionally including ports. public RabbitMQMessageBusOptionsBuilder Hosts(params string[] hosts) { Target.Hosts = hosts ?? throw new ArgumentNullException(nameof(hosts)); return this; } - /// - /// Sets the list of hosts for failover support. Replaces the host from ConnectionString. - /// - /// Hostnames in format "hostname" or "hostname:port" + /// Replace the URI host. An empty list uses the URI endpoint. public RabbitMQMessageBusOptionsBuilder Hosts(IList hosts) { Target.Hosts = hosts ?? throw new ArgumentNullException(nameof(hosts)); @@ -318,7 +366,7 @@ public RabbitMQMessageBusOptionsBuilder PrefetchSize(uint prefetchSize) return this; } - [Obsolete("Global QoS is deprecated in RabbitMQ 4.3+ and will be removed in a future version. Use per-channel prefetch (GlobalQos = false) instead.")] + [Obsolete("Global QoS is deprecated in RabbitMQ 4.3+ and will be removed in a future version. Use per-consumer prefetch (GlobalQos = false) instead.")] public RabbitMQMessageBusOptionsBuilder GlobalQos(bool globalQos) { #pragma warning disable CS0618 @@ -332,24 +380,47 @@ public RabbitMQMessageBusOptionsBuilder DeliveryLimit(long deliveryLimit) Target.DeliveryLimit = deliveryLimit; Target.Arguments ??= new Dictionary(); - Target.Arguments["x-delivery-limit"] = deliveryLimit; + Target.Arguments[RabbitMQConstants.DeliveryLimitArgument] = deliveryLimit; return this; } - /// - /// Enables publisher confirms, which guarantees the message reached the broker. - /// When enabled, PublishAsync waits for broker confirmation before returning. - /// See: https://www.rabbitmq.com/docs/confirms#publisher-confirms - /// - /// Whether to enable publisher confirms. Default: true. - /// The builder instance for method chaining. + /// Enable broker confirmation, not consumer-processing confirmation. public RabbitMQMessageBusOptionsBuilder PublisherConfirmsEnabled(bool enabled = true) { Target.PublisherConfirmsEnabled = enabled; return this; } + /// Require a confirmed, routed immediate publication; scheduled routing is not covered. + public RabbitMQMessageBusOptionsBuilder RequirePublishRouting(bool required = true) + { + Target.RequirePublishRouting = required; + return this; + } + + /// Require matching live handlers to complete successfully before acknowledgement. + public RabbitMQMessageBusOptionsBuilder RequireSuccessfulDispatch(bool required = true) + { + Target.RequireSuccessfulDispatch = required; + return this; + } + + /// Reject delayed publications instead of using an in-memory fallback. + public RabbitMQMessageBusOptionsBuilder RequireBrokerDelayedDelivery(bool required = true) + { + Target.RequireBrokerDelayedDelivery = required; + return this; + } + + /// Set the positive, bounded transport shutdown timeout. + public RabbitMQMessageBusOptionsBuilder ShutdownTimeout(TimeSpan timeout) + { + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeout, TimeSpan.Zero); + Target.ShutdownTimeout = timeout; + return this; + } + /// /// Sets the maximum time each publish attempt will wait for connection recovery before failing. /// During a transient connection drop, publishes suspend and resume automatically on recovery. @@ -365,12 +436,10 @@ public RabbitMQMessageBusOptionsBuilder PublishRecoveryTimeout(TimeSpan timeout) } /// - /// Configures the message bus for RabbitMQ quorum queues using recommended settings. - /// Disables auto-delete and exclusive mode for subscription queues, - /// and sets quorum-specific arguments based on the current DeliveryLimit value. - /// Preserves any existing arguments. + /// Select quorum queues and retain them across disconnects. Preserves other raw arguments. + /// The broker delivery limit is set to the current DeliveryLimit; an explicit raw override + /// can instead disable that broker limit (-1) while retaining the application retry budget. /// - /// The builder instance for method chaining. public RabbitMQMessageBusOptionsBuilder UseQuorumQueues() { if (Target.MaxPriority.HasValue) @@ -380,11 +449,8 @@ public RabbitMQMessageBusOptionsBuilder UseQuorumQueues() Target.IsSubscriptionQueueExclusive = false; Target.Arguments ??= new Dictionary(); - - // Add or update quorum-specific arguments - Target.Arguments["x-queue-type"] = "quorum"; - Target.Arguments["x-delivery-limit"] = Target.DeliveryLimit; - + Target.Arguments[RabbitMQConstants.QueueTypeArgument] = "quorum"; + Target.Arguments[RabbitMQConstants.DeliveryLimitArgument] = Target.DeliveryLimit; return this; } @@ -392,7 +458,7 @@ public RabbitMQMessageBusOptionsBuilder UseQuorumQueues() /// Sets the heartbeat timeout negotiated with the broker. /// Controls how quickly dead TCP connections are detected. /// - /// Heartbeat interval. TimeSpan.Zero disables heartbeats. + /// Requested heartbeat timeout. Heartbeats are disabled only when both peers offer zero. public RabbitMQMessageBusOptionsBuilder RequestedHeartbeat(TimeSpan heartbeat) { ArgumentOutOfRangeException.ThrowIfLessThan(heartbeat, TimeSpan.Zero); @@ -411,12 +477,7 @@ public RabbitMQMessageBusOptionsBuilder NetworkRecoveryInterval(TimeSpan interva return this; } - /// - /// Configures a dead letter exchange for messages that exceed the delivery limit or are rejected. - /// - /// The DLX exchange name. - /// Optional routing key for dead-lettered messages. - /// Dead-letter strategy. AtLeastOnce requires overflow to be RejectPublish. + /// Configure terminal routing. Provision the durable destination separately. public RabbitMQMessageBusOptionsBuilder DeadLetterExchange(string exchange, string? routingKey = null, DeadLetterStrategy? strategy = null) { ArgumentException.ThrowIfNullOrWhiteSpace(exchange); @@ -450,11 +511,7 @@ public RabbitMQMessageBusOptionsBuilder ConsumerTimeout(TimeSpan timeout) return this; } - /// - /// Enables single active consumer mode for strict message ordering with automatic failover. - /// Only one consumer at a time will receive messages; others act as standby. - /// - /// Whether to enable single active consumer. Default: true. + /// Enable a single active consumer; does not eliminate redelivery or enforce business ordering. public RabbitMQMessageBusOptionsBuilder UseSingleActiveConsumer(bool enabled = true) { Target.SingleActiveConsumer = enabled; @@ -481,7 +538,7 @@ public RabbitMQMessageBusOptionsBuilder UseMessagePriority(byte maxPriority = 32 /// /// Configures native delayed retry for quorum queues (RabbitMQ 4.3+). /// Rejected/failed messages are held in a delayed state with linear backoff before redelivery. - /// This replaces the need for the delayed message exchange plugin for retry scenarios. + /// This applies to rejected messages; it does not schedule initial publications. /// /// Minimum delay in milliseconds (multiplied by delivery count). /// Maximum delay cap in milliseconds. @@ -492,7 +549,7 @@ public RabbitMQMessageBusOptionsBuilder UseDelayedRetries(int minDelayMs = 1000, ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(maxDelayMs, 0); if (maxDelayMs < minDelayMs) - throw new ArgumentOutOfRangeException(nameof(maxDelayMs), $"maxDelayMs ({maxDelayMs}) must be >= minDelayMs ({minDelayMs})"); + throw new ArgumentOutOfRangeException(nameof(maxDelayMs), FormattableString.Invariant($"maxDelayMs ({maxDelayMs}) must be >= minDelayMs ({minDelayMs})")); Target.DelayedRetryType = retryType; Target.DelayedRetryMin = minDelayMs; diff --git a/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageConverter.cs b/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageConverter.cs new file mode 100644 index 00000000..c2fb0946 --- /dev/null +++ b/src/Foundatio.RabbitMQ/Messaging/RabbitMQMessageConverter.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using Foundatio.Utility; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; + +namespace Foundatio.Messaging; + +internal static class RabbitMQMessageConverter +{ + private const string PriorityPropertyKey = "Priority"; + + internal static IMessage Convert(BasicDeliverEventArgs envelope, Type? type, Func deserialize) + { + // A handler can outlive the transport callback after cancellation, so own the payload. + var message = new Message(envelope.Body.ToArray(), deserialize) + { + Type = envelope.BasicProperties.Type, + ClrType = type, + CorrelationId = envelope.BasicProperties.CorrelationId, + UniqueId = envelope.BasicProperties.MessageId + }; + if (envelope.BasicProperties.Headers is not null) + foreach (var header in envelope.BasicProperties.Headers) + { + if (header.Value is byte[] bytes) + message.Properties[header.Key] = Encoding.UTF8.GetString(bytes); + else if (header.Value is not null && System.Convert.ToString(header.Value, CultureInfo.InvariantCulture) is { } value) + message.Properties[header.Key] = value; + } + return message; + } + + internal static BasicProperties CreateProperties(string messageType, MessageOptions options, RabbitMQMessageBusOptions settings) + { + bool delayed = options.DeliveryDelay.GetValueOrDefault() > TimeSpan.Zero; + var properties = new BasicProperties + { + MessageId = options.UniqueId ?? Guid.NewGuid().ToString("N"), + CorrelationId = options.CorrelationId, + Type = messageType, + Persistent = settings.IsDurable + }; + if (settings.DefaultMessageTimeToLive.HasValue) + properties.Expiration = settings.DefaultMessageTimeToLive.Value.TotalMilliseconds.ToString(CultureInfo.InvariantCulture); + if (options.Properties.TryGetValue(PriorityPropertyKey, out string? priorityValue) && Byte.TryParse(priorityValue, NumberStyles.Integer, CultureInfo.InvariantCulture, out byte priority)) + properties.Priority = priority; + if (options.Properties.Count > 0) + { + properties.Headers = new Dictionary(); + foreach (var property in options.Properties) + if (!String.Equals(property.Key, PriorityPropertyKey, StringComparison.Ordinal)) + properties.Headers.Add(property.Key, property.Value); + } + if (delayed) + { + double delayMs = options.DeliveryDelay!.Value.TotalMilliseconds; + if (delayMs > Int32.MaxValue) + throw new ArgumentOutOfRangeException(nameof(options), "DeliveryDelay exceeds the supported delayed-exchange range."); + properties.Headers ??= new Dictionary(); + properties.Headers[RabbitMQConstants.XDelayHeader] = (int)delayMs; + } + return properties; + } + + internal static BasicProperties CopyHandoffProperties(BasicDeliverEventArgs envelope) + { + var properties = new BasicProperties(envelope.BasicProperties) + { + UserId = null, + Headers = envelope.BasicProperties.Headers is { } headers + ? new Dictionary(headers) : new Dictionary() + }; + properties.Headers.Remove(RabbitMQConstants.CarbonCopyHeader); + properties.Headers.Remove(RabbitMQConstants.BlindCarbonCopyHeader); + properties.Headers.Remove(RabbitMQConstants.XDelayHeader); + if (!properties.Headers.ContainsKey(RabbitMQConstants.XOriginalMessageIdHeader)) + properties.Headers[RabbitMQConstants.XOriginalMessageIdHeader] = properties.MessageId; + return properties; + } + + internal static long GetRetryCountFromHeader(BasicDeliverEventArgs envelope) + { + if (envelope.BasicProperties.Headers?.TryGetValue(RabbitMQConstants.XDeliveryCountHeader, out object? value) is not true) + return 0; + if (value is long number) + return number >= 0 ? number : Int64.MaxValue; + string? text = value is byte[] bytes ? Encoding.UTF8.GetString(bytes) : System.Convert.ToString(value, CultureInfo.InvariantCulture); + // Invalid/overflowing metadata must not reset an exhausted budget. + return Int64.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out long count) && count >= 0 + ? count : Int64.MaxValue; + } + +} diff --git a/src/Foundatio.RabbitMQ/Messaging/RabbitMQTopology.cs b/src/Foundatio.RabbitMQ/Messaging/RabbitMQTopology.cs new file mode 100644 index 00000000..832f2705 --- /dev/null +++ b/src/Foundatio.RabbitMQ/Messaging/RabbitMQTopology.cs @@ -0,0 +1,175 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Extensions; +using Foundatio.Utility; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Foundatio.Messaging; + +internal sealed class RabbitMQTopology +{ + private static readonly Version _delayedExchangePluginIncompatibleVersion = new(4, 3); + private static readonly Version _globalQosRemovedVersion = new(4, 3); + private readonly RabbitMQMessageBusOptions _options; + private readonly ILogger _logger; + private bool? _delayedExchangePluginEnabled; + private readonly bool _isQuorumQueue; + + internal bool IsQuorumQueue => _isQuorumQueue; + internal bool SupportsDelayedExchange => _delayedExchangePluginEnabled is true; + + internal RabbitMQTopology(RabbitMQMessageBusOptions options, ILogger logger) + { + _options = options; + _logger = logger; + _isQuorumQueue = RabbitMQMessageBusOptions.IsQuorumQueue(options.Arguments); + } + + internal Version? DetectServerVersion(IConnection connection) + { + var version = ParseServerVersion(connection.ServerProperties); + if (version is not null) + _logger.LogDebug("Connected to RabbitMQ server version {ServerVersion}", version); + return version; + } + + /// Parse the broker version from the UTF-8 AMQP server property. + internal static Version? ParseServerVersion(IDictionary? serverProperties) + { + if (serverProperties?.TryGetValue("version", out var value) is not true || value is not byte[] bytes) + return null; + return Version.TryParse(Encoding.UTF8.GetString(bytes), out var version) ? version : null; + } + + internal async Task CreateDelayedExchangeAsync(IChannel channel, Version? serverVersion, CancellationToken cancellationToken) + { + // RabbitMQ 4.3 removed the Mnesia store required by the archived plugin. + // Probe earlier brokers (and unknown versions) to retain 4.2 delayed delivery. + if (serverVersion is not null && serverVersion >= _delayedExchangePluginIncompatibleVersion) + { + if (_delayedExchangePluginEnabled is not false) + _logger.LogWarning("The delayed-message exchange plugin is incompatible with RabbitMQ {ServerVersion}; delayed publications use in-process scheduling unless broker delivery is required", serverVersion); + _delayedExchangePluginEnabled = false; + return null; + } + try + { + await channel.ExchangeDeclareAsync(_options.Topic, "x-delayed-message", _options.IsDurable, false, + new Dictionary { [RabbitMQConstants.DelayedExchangeTypeArgument] = ExchangeType.Fanout }, cancellationToken: cancellationToken).AnyContext(); + if (_delayedExchangePluginEnabled is not true) + _logger.LogWarning("The delayed-message exchange plugin is archived and incompatible with RabbitMQ 4.3+; review its performance and migration guidance before upgrading the broker"); + _delayedExchangePluginEnabled = true; + return true; + } + catch (OperationInterruptedException exception) when (IsRegularExchangeFallback(exception)) + { + // Unsupported plugin type, or an existing regular fanout topic, is a known + // non-scheduling topology. Other declaration/permission/network errors propagate. + _delayedExchangePluginEnabled = false; + return false; + } + } + + private static bool IsRegularExchangeFallback(OperationInterruptedException exception) + { + var reason = exception.ShutdownReason; + if (reason is null || reason.ReplyCode is not (406 or 503)) + return false; + return reason.ReplyText.Contains("unknown exchange type 'x-delayed-message'", StringComparison.OrdinalIgnoreCase) + || (reason.ReplyCode == 406 + && reason.ReplyText.Contains("inequivalent arg 'type'", StringComparison.OrdinalIgnoreCase) + && reason.ReplyText.Contains("current is 'fanout'", StringComparison.OrdinalIgnoreCase)); + } + + internal Task DeclareSubscriptionExchangeAsync(IChannel channel, CancellationToken cancellationToken) => + _delayedExchangePluginEnabled is true + ? channel.ExchangeDeclareAsync(_options.Topic, "x-delayed-message", _options.IsDurable, false, + new Dictionary { [RabbitMQConstants.DelayedExchangeTypeArgument] = ExchangeType.Fanout }, cancellationToken: cancellationToken) + : CreateRegularExchangeAsync(channel, cancellationToken); + + internal Task CreateRegularExchangeAsync(IChannel channel, CancellationToken cancellationToken) => + channel.ExchangeDeclareAsync(_options.Topic, ExchangeType.Fanout, _options.IsDurable, false, cancellationToken: cancellationToken); + + internal async Task CreateQueueAsync(IChannel channel, Version? serverVersion, CancellationToken cancellationToken) + { + var arguments = _options.Arguments is not null + ? new Dictionary(_options.Arguments) : new Dictionary(); + bool isQuorumQueue = RabbitMQMessageBusOptions.IsQuorumQueue(arguments); + if (isQuorumQueue != _isQuorumQueue) + throw new InvalidOperationException("Queue type cannot change after the message bus is constructed."); + if (isQuorumQueue && _options.MaxPriority.HasValue) + throw new InvalidOperationException("MaxPriority applies only to classic queues and cannot be used with quorum queues."); + + if (_isQuorumQueue && !arguments.ContainsKey(RabbitMQConstants.DeliveryLimitArgument)) + arguments[RabbitMQConstants.DeliveryLimitArgument] = _options.DeliveryLimit; + if (!String.IsNullOrWhiteSpace(_options.DeadLetterExchange)) + { + arguments[RabbitMQConstants.DeadLetterExchangeArgument] = _options.DeadLetterExchange; + if (_options.DeadLetterRoutingKey is not null) + arguments[RabbitMQConstants.DeadLetterRoutingKeyArgument] = _options.DeadLetterRoutingKey; + if (_options.DeadLetterStrategy.HasValue) + { + if (_options.DeadLetterStrategy == DeadLetterStrategy.AtLeastOnce) + { + if (!_isQuorumQueue) + throw new MessageBusException("At-least-once broker dead-lettering requires quorum queues."); + if (_options.Overflow != QueueOverflowBehavior.RejectPublish) + throw new MessageBusException("At-least-once broker dead-lettering requires RejectPublish overflow."); + } + arguments[RabbitMQConstants.DeadLetterStrategyArgument] = _options.DeadLetterStrategy.Value.ToEnumString(); + } + } + if (_options.Overflow.HasValue) + arguments[RabbitMQConstants.OverflowArgument] = _options.Overflow.Value.ToEnumString(); + if (_options.ConsumerTimeout.HasValue) + { + if (!_isQuorumQueue) + throw new MessageBusException("Per-queue consumer timeout (x-consumer-timeout) requires quorum queues (RabbitMQ 4.3+). Call UseQuorumQueues() before ConsumerTimeout()."); + if (serverVersion is not null && serverVersion < _delayedExchangePluginIncompatibleVersion) + throw new MessageBusException(FormattableString.Invariant($"Per-queue consumer timeout (x-consumer-timeout) requires RabbitMQ 4.3+. Detected server version: {serverVersion}.")); + arguments[RabbitMQConstants.ConsumerTimeoutArgument] = (long)_options.ConsumerTimeout.Value.TotalMilliseconds; + } + if (_options.SingleActiveConsumer) + arguments[RabbitMQConstants.SingleActiveConsumerArgument] = true; + if (_options.MaxPriority.HasValue) + arguments[RabbitMQConstants.MaxPriorityArgument] = (int)_options.MaxPriority.Value; + if (_options.DelayedRetryType.HasValue) + { + if (!_isQuorumQueue) + throw new MessageBusException("Delayed retries (x-delayed-retry-*) require quorum queues (RabbitMQ 4.3+). Call UseQuorumQueues() before UseDelayedRetries()."); + if (serverVersion is not null && serverVersion < _delayedExchangePluginIncompatibleVersion) + throw new MessageBusException(FormattableString.Invariant($"Delayed retries (x-delayed-retry-*) require RabbitMQ 4.3+. Detected server version: {serverVersion}.")); + arguments[RabbitMQConstants.DelayedRetryTypeArgument] = _options.DelayedRetryType.Value.ToEnumString(); + if (_options.DelayedRetryMin.HasValue) + arguments[RabbitMQConstants.DelayedRetryMinArgument] = _options.DelayedRetryMin.Value; + if (_options.DelayedRetryMax.HasValue) + arguments[RabbitMQConstants.DelayedRetryMaxArgument] = _options.DelayedRetryMax.Value; + } + var queue = await channel.QueueDeclareAsync(_options.SubscriptionQueueName, _options.IsDurable, + _options.IsSubscriptionQueueExclusive, _options.SubscriptionQueueAutoDelete, + arguments.Count > 0 ? arguments : null, cancellationToken: cancellationToken).AnyContext(); + await channel.QueueBindAsync(queue.QueueName, _options.Topic, String.Empty, cancellationToken: cancellationToken).AnyContext(); + return queue.QueueName; + } + + internal async Task ConfigurePrefetchAsync(IChannel channel, Version? serverVersion, CancellationToken cancellationToken) + { + if (_options.PrefetchCount > 0 || _options.PrefetchSize > 0) + { +#pragma warning disable CS0618 + bool global = _options.GlobalQos; +#pragma warning restore CS0618 + if (global && (_isQuorumQueue || (serverVersion is not null && serverVersion >= _globalQosRemovedVersion))) + { + _logger.LogWarning("Global QoS is unavailable; using per-consumer prefetch"); + global = false; + } + await channel.BasicQosAsync(_options.PrefetchSize, _options.PrefetchCount, global, cancellationToken).AnyContext(); + } + } +} diff --git a/src/Foundatio.RabbitMQ/Utility/RabbitMQConstants.cs b/src/Foundatio.RabbitMQ/Utility/RabbitMQConstants.cs new file mode 100644 index 00000000..44eef786 --- /dev/null +++ b/src/Foundatio.RabbitMQ/Utility/RabbitMQConstants.cs @@ -0,0 +1,74 @@ +namespace Foundatio.Utility; + +/// RabbitMQ wire header and queue argument names shared by publishers, consumers, and topology setup. +public static class RabbitMQConstants +{ + /// The x-delivery-count wire name. + public const string XDeliveryCountHeader = "x-delivery-count"; + + /// The x-original-message-id wire name. + public const string XOriginalMessageIdHeader = "x-original-message-id"; + + /// The x-delay wire name. + public const string XDelayHeader = "x-delay"; + + /// The x-original-expiration wire name. + public const string XOriginalExpirationHeader = "x-original-expiration"; + + /// The x-foundatio-failure-type wire name. + public const string FailureTypeHeader = "x-foundatio-failure-type"; + + /// The x-foundatio-original-exchange wire name. + public const string OriginalExchangeHeader = "x-foundatio-original-exchange"; + + /// The x-foundatio-original-routing-key wire name. + public const string OriginalRoutingKeyHeader = "x-foundatio-original-routing-key"; + + /// The CC wire name. + public const string CarbonCopyHeader = "CC"; + + /// The BCC wire name. + public const string BlindCarbonCopyHeader = "BCC"; + + /// The x-queue-type wire name. + public const string QueueTypeArgument = "x-queue-type"; + + /// The x-delivery-limit wire name. + public const string DeliveryLimitArgument = "x-delivery-limit"; + + /// The x-dead-letter-exchange wire name. + public const string DeadLetterExchangeArgument = "x-dead-letter-exchange"; + + /// The x-dead-letter-routing-key wire name. + public const string DeadLetterRoutingKeyArgument = "x-dead-letter-routing-key"; + + /// The x-dead-letter-strategy wire name. + public const string DeadLetterStrategyArgument = "x-dead-letter-strategy"; + + /// The x-overflow wire name. + public const string OverflowArgument = "x-overflow"; + + /// The x-consumer-timeout wire name. + public const string ConsumerTimeoutArgument = "x-consumer-timeout"; + + /// The x-single-active-consumer wire name. + public const string SingleActiveConsumerArgument = "x-single-active-consumer"; + + /// The x-max-priority wire name. + public const string MaxPriorityArgument = "x-max-priority"; + + /// The x-delayed-retry-type wire name. + public const string DelayedRetryTypeArgument = "x-delayed-retry-type"; + + /// The x-delayed-retry-min wire name. + public const string DelayedRetryMinArgument = "x-delayed-retry-min"; + + /// The x-delayed-retry-max wire name. + public const string DelayedRetryMaxArgument = "x-delayed-retry-max"; + + /// The x-delayed-type wire name. + public const string DelayedExchangeTypeArgument = "x-delayed-type"; + + /// The x-max-length-bytes wire name. + public const string MaxLengthBytesArgument = "x-max-length-bytes"; +} diff --git a/tests/Foundatio.RabbitMQ.ProcessProbe/Directory.Build.props b/tests/Foundatio.RabbitMQ.ProcessProbe/Directory.Build.props new file mode 100644 index 00000000..64839310 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.ProcessProbe/Directory.Build.props @@ -0,0 +1,9 @@ + + + net10.0 + latest + enable + true + false + + diff --git a/tests/Foundatio.RabbitMQ.ProcessProbe/Foundatio.RabbitMQ.ProcessProbe.csproj b/tests/Foundatio.RabbitMQ.ProcessProbe/Foundatio.RabbitMQ.ProcessProbe.csproj new file mode 100644 index 00000000..3d145609 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.ProcessProbe/Foundatio.RabbitMQ.ProcessProbe.csproj @@ -0,0 +1,8 @@ + + + Exe + + + + + diff --git a/tests/Foundatio.RabbitMQ.ProcessProbe/Program.cs b/tests/Foundatio.RabbitMQ.ProcessProbe/Program.cs new file mode 100644 index 00000000..f95dfe53 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.ProcessProbe/Program.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; + +// Test-only child process. A parent integration test kills this process after the +// broker confirms a delayed publication and before the delivery deadline. +using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(2)); +string connectionString = Required("FOUNDATIO_PROBE_URI"); +var uri = new Uri(connectionString); +if (!uri.IsLoopback || uri.Scheme is not ("amqp" or "amqps")) + throw new InvalidOperationException("The process probe requires a test-owned loopback broker."); +string topic = Required("FOUNDATIO_PROBE_TOPIC"); +string id = Required("FOUNDATIO_PROBE_ID"); +string checkpoint = Required("FOUNDATIO_PROBE_CHECKPOINT"); +if (!Path.IsPathFullyQualified(checkpoint)) + throw new InvalidOperationException("The test checkpoint must have an absolute path."); + +await using var bus = new RabbitMQMessageBus(new RabbitMQMessageBusOptions +{ + ConnectionString = connectionString, + Topic = topic, + IsDurable = true, + PublisherConfirmsEnabled = true, + RequireBrokerDelayedDelivery = true +}); +var startedAt = DateTimeOffset.UtcNow; +await bus.PublishAsync(new ProcessMessage { Id = id }, new MessageOptions +{ + UniqueId = id, + DeliveryDelay = TimeSpan.FromSeconds(30) +}, timeout.Token); +string temporary = checkpoint + ".tmp"; +await File.WriteAllTextAsync(temporary, JsonSerializer.Serialize(new { Id = id, StartedAt = startedAt }), timeout.Token); +File.Move(temporary, checkpoint); +Console.WriteLine($"Broker accepted delayed event {id}; parent may terminate this process."); +await Task.Delay(Timeout.InfiniteTimeSpan, timeout.Token); + +static string Required(string name) => Environment.GetEnvironmentVariable(name) + ?? throw new InvalidOperationException($"Missing process-probe setting {name}."); + +public sealed class ProcessMessage +{ + public string Id { get; set; } = String.Empty; +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Foundatio.RabbitMQ.Tests.csproj b/tests/Foundatio.RabbitMQ.Tests/Foundatio.RabbitMQ.Tests.csproj index c065af8a..0ce1f4c0 100644 --- a/tests/Foundatio.RabbitMQ.Tests/Foundatio.RabbitMQ.Tests.csproj +++ b/tests/Foundatio.RabbitMQ.Tests/Foundatio.RabbitMQ.Tests.csproj @@ -2,6 +2,7 @@ + diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqDelayedProcessTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqDelayedProcessTests.cs new file mode 100644 index 00000000..6438629c --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqDelayedProcessTests.cs @@ -0,0 +1,118 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Xunit; +using Microsoft.Extensions.Logging; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +[Collection(nameof(RabbitMqTestCollection))] +public class RabbitMqDelayedProcessTests(AspireFixture fixture, ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Fact] + public async Task PublishAsync_WithBrokerDelayAndPublisherProcessTermination_DeliversScheduledMessageAsync() + { + Assert.SkipWhen(String.IsNullOrEmpty(fixture.MessagingDelayedConnectionString), "Delayed broker unavailable"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(90)); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingDelayedConnectionString!, Log, token); + var options = context.Options(); + await using (var provision = new RabbitMQMessageBus(options)) + await provision.SubscribeAsync(_ => { }, token); + string id = Guid.NewGuid().ToString("N"); + string directory = Path.Combine(Path.GetTempPath(), $"foundatio-delayed-process-{id}"); + Directory.CreateDirectory(directory); + string checkpoint = Path.Combine(directory, "accepted.json"); + string helper = GetProcessProbePath(); + var start = new ProcessStartInfo("dotnet") + { + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + start.ArgumentList.Add(helper); + // Do not put broker credentials in command-line arguments or test output. + start.Environment["FOUNDATIO_PROBE_URI"] = fixture.MessagingDelayedConnectionString; + start.Environment["FOUNDATIO_PROBE_TOPIC"] = context.Topic; + start.Environment["FOUNDATIO_PROBE_ID"] = id; + start.Environment["FOUNDATIO_PROBE_CHECKPOINT"] = checkpoint; + using var process = Process.Start(start) ?? throw new InvalidOperationException("The test publisher process did not start."); + var stdout = process.StandardOutput.ReadToEndAsync(token); + var stderr = process.StandardError.ReadToEndAsync(token); + try + { + while (!File.Exists(checkpoint)) + { + Assert.False(process.HasExited, "The publisher exited before confirming its scheduled event."); + await Task.Delay(TimeSpan.FromMilliseconds(50), token); + } + using var accepted = JsonDocument.Parse(await File.ReadAllTextAsync(checkpoint, token)); + Assert.Equal(id, accepted.RootElement.GetProperty("Id").GetString()); + var started = accepted.RootElement.GetProperty("StartedAt").GetDateTimeOffset(); + Assert.False(process.HasExited); + // Act + process.Kill(entireProcessTree: true); + await process.WaitForExitAsync(token); + // Assert + Assert.True(process.HasExited); + Assert.True(DateTimeOffset.UtcNow < started.AddSeconds(30), "The process must be killed before its delayed event is due."); + _logger.LogInformation("Killed publisher after confirmed scheduling and before due time: event={MessageId}, process={ProcessId}", + id, process.Id); + + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var subscriber = new RabbitMQMessageBus(options); + await subscriber.SubscribeAsync(message => + { + Assert.Contains(id, Encoding.UTF8.GetString(message.Data.Span)); + received.TrySetResult(message.UniqueId!); + }, token); + Assert.Equal(id, await received.Task.WaitAsync(token)); + await RabbitMqReliabilityTestContext.WaitAsync(() => subscriber.ActiveDeliveryCount == 0, token); + _logger.LogInformation("Broker delivered scheduled event after publisher process termination: {MessageId}", + id); + } + finally + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await process.WaitForExitAsync(cleanup.Token); + } + try + { + _logger.LogInformation("Publisher standard output: {Output}", await stdout); + _logger.LogInformation("Publisher standard error: {Error}", await stderr); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + } + Directory.Delete(directory, recursive: true); + } + } + + private static string GetProcessProbePath() + { + var testOutput = new DirectoryInfo(AppContext.BaseDirectory); + string configuration = testOutput.Parent?.Name + ?? throw new InvalidOperationException("Cannot locate the test build configuration."); + DirectoryInfo? root = testOutput; + while (root is not null && !File.Exists(Path.Combine(root.FullName, "Foundatio.RabbitMQ.slnx"))) + root = root.Parent; + Assert.NotNull(root); + string helper = Path.Combine(root.FullName, "tests", "Foundatio.RabbitMQ.ProcessProbe", "bin", configuration, + "net10.0", "Foundatio.RabbitMQ.ProcessProbe.dll"); + Assert.True(File.Exists(helper), "Build the solution, including its process probe, before running integration tests."); + return helper; + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqDeliverySafetyTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqDeliverySafetyTests.cs new file mode 100644 index 00000000..dcf636b7 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqDeliverySafetyTests.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; +using Foundatio.Utility; +using Foundatio.Xunit; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +[Collection(nameof(RabbitMqTestCollection))] +public class RabbitMqDeliverySafetyTests(AspireFixture fixture, ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Fact] + public async Task PublishAsync_AfterChannelClosure_RecreatesPublisherChannelAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + string topic = $"safety-publisher-{Guid.NewGuid():N}"; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var bus = CreateBus(topic, "subscriber"); + await bus.SubscribeAsync(message => received.TrySetResult(message.Data!), timeout.Token); + var connection = GetField(bus, "_publisherConnection"); + // Act + await GetField(bus, "_publisherChannel").CloseAsync(200, "Synthetic publisher channel fault", cancellationToken: timeout.Token); + Assert.True(connection.IsOpen); + await bus.PublishAsync(new SimpleMessageA { Data = topic }, cancellationToken: timeout.Token); + // Assert + Assert.Equal(topic, await received.Task.WaitAsync(timeout.Token)); + } + + [Fact] + public async Task PublishAsync_WithDeletedDelayedExchange_RecreatesItAfterChannelClosureAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(40)); + string topic = $"safety-delayed-{Guid.NewGuid():N}"; + var factory = new ConnectionFactory { Uri = new Uri(fixture.MessagingDelayedConnectionString!) }; + await using var connection = await factory.CreateConnectionAsync(timeout.Token); + await using var admin = await connection.CreateChannelAsync(cancellationToken: timeout.Token); + await using var bus = new RabbitMQMessageBus(new RabbitMQMessageBusOptions + { + ConnectionString = fixture.MessagingDelayedConnectionString, + Topic = topic, + IsDurable = true, + PublisherConfirmsEnabled = true, + RequireBrokerDelayedDelivery = true, + LoggerFactory = Log + }); + var options = new MessageOptions { DeliveryDelay = TimeSpan.FromMinutes(1) }; + try + { + await bus.PublishAsync(new SimpleMessageA { Data = "first" }, options, timeout.Token); + await admin.ExchangeDeleteAsync(topic, cancellationToken: timeout.Token); + + // Act + await GetField(bus, "_publisherChannel").CloseAsync(200, "Synthetic channel fault", cancellationToken: timeout.Token); + await bus.PublishAsync(new SimpleMessageA { Data = "second" }, options, timeout.Token); + + // Assert + await admin.ExchangeDeclarePassiveAsync(topic, cancellationToken: timeout.Token); + Assert.True(admin.IsOpen); + } + finally + { + await admin.ExchangeDeleteAsync(topic, cancellationToken: timeout.Token); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_AfterChannelClosureOrConsumerCancellation_ResumesWithoutRestartAsync(bool cancelConsumer) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(40)); + string topic = $"safety-lifecycle-{Guid.NewGuid():N}"; + var received = new ConcurrentDictionary(); + await using var bus = CreateBus(topic, "consumer"); + await bus.SubscribeAsync(message => received.TryAdd(message.Data!, 0), timeout.Token); + await bus.PublishAsync(new SimpleMessageA { Data = "before" }, cancellationToken: timeout.Token); + await WaitAsync(() => received.ContainsKey("before"), timeout.Token); + var connection = GetField(bus, "_subscriberConnection"); + var channel = GetField(bus, "_subscriberChannel"); + var consumer = GetField(bus, "_consumer"); + // Act + if (cancelConsumer) + await channel.BasicCancelAsync(Assert.Single(consumer.ConsumerTags), cancellationToken: timeout.Token); + else + await channel.CloseAsync(200, "Synthetic channel-only fault", cancellationToken: timeout.Token); + Assert.True(connection.IsOpen, "The test must not substitute TCP/network recovery for a channel/consumer fault."); + + await WaitAsync(() => + { + var current = typeof(RabbitMQMessageBus).GetField("_consumer", BindingFlags.Instance | BindingFlags.NonPublic)?.GetValue(bus) as AsyncEventingBasicConsumer; + return current is not null && !ReferenceEquals(consumer, current) && current.IsRunning; + }, timeout.Token); + await bus.PublishAsync(new SimpleMessageA { Data = "after" }, cancellationToken: timeout.Token); + await WaitAsync(() => received.ContainsKey("after"), timeout.Token); + // Assert + Assert.Equal(new[] { "after", "before" }, received.Keys.OrderBy(value => value)); + } + + [Fact] + public async Task SubscribeAsync_WhenQueueTypeChangesAfterConstruction_RejectsPriorityAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + string topic = $"safety-priority-options-{Guid.NewGuid():N}"; + var arguments = new Dictionary { [RabbitMQConstants.QueueTypeArgument] = "classic" }; + await using var bus = new RabbitMQMessageBus(new RabbitMQMessageBusOptions + { + ConnectionString = fixture.MessagingConnectionString, + LoggerFactory = Log, + Topic = topic, + SubscriptionQueueName = $"{topic}-consumer", + MaxPriority = 3, + Arguments = arguments + }); + var factory = new ConnectionFactory { Uri = new Uri(fixture.MessagingConnectionString!) }; + await using var adminConnection = await factory.CreateConnectionAsync(timeout.Token); + await using var admin = await adminConnection.CreateChannelAsync(cancellationToken: timeout.Token); + try + { + arguments[RabbitMQConstants.QueueTypeArgument] = "quorum"; + + // Act + var exception = await Assert.ThrowsAsync(() => + bus.SubscribeAsync(_ => { }, timeout.Token)); + + // Assert + Assert.Contains("queue type", exception.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + await admin.ExchangeDeleteAsync(topic, cancellationToken: timeout.Token); + } + } + + [Fact] + public async Task SubscribeAsync_WithFailingHandler_RetriesOnlyFailedSubscriptionWithStableIdentityAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + string topic = $"safety-retry-{Guid.NewGuid():N}"; + string id = Guid.NewGuid().ToString("N"); + var successes = new ConcurrentQueue(); + var retries = new ConcurrentQueue(); + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var successful = CreateBus(topic, "successful"); + await using var failing = CreateBus(topic, "failing"); + await successful.SubscribeAsync>(message => successes.Enqueue(message.UniqueId!), timeout.Token); + await failing.SubscribeAsync>(message => + { + retries.Enqueue(message.UniqueId!); + if (retries.Count == 1) + throw new InvalidOperationException("Synthetic first-attempt failure"); + done.TrySetResult(); + }, timeout.Token); + + // Act + await successful.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, timeout.Token); + await done.Task.WaitAsync(timeout.Token); + // Complete the transport shutdown before inspecting the final callback inventory. + await failing.DisposeAsync(); + await successful.DisposeAsync(); + // Assert + Assert.Equal(new[] { id }, successes.ToArray()); + Assert.Equal(new[] { id, id }, retries.ToArray()); + } + + private RabbitMQMessageBus CreateBus(string topic, string suffix) => new(new RabbitMQMessageBusOptions + { + ConnectionString = fixture.MessagingConnectionString, + LoggerFactory = Log, + Topic = topic, + SubscriptionQueueName = $"{topic}-{suffix}", + IsDurable = false, + IsSubscriptionQueueExclusive = false, + SubscriptionQueueAutoDelete = true, + AcknowledgementStrategy = AcknowledgementStrategy.Automatic, + PrefetchCount = 1, + PublisherConfirmsEnabled = true, + NetworkRecoveryInterval = TimeSpan.FromSeconds(1), + Arguments = new Dictionary { ["x-queue-type"] = "classic" } + }); + + private static T GetField(RabbitMQMessageBus bus, string name) + { + var field = typeof(RabbitMQMessageBus).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsAssignableFrom(field.GetValue(bus)); + } + + private static async Task WaitAsync(Func condition, CancellationToken cancellationToken) + { + while (!condition()) + await Task.Delay(TimeSpan.FromMilliseconds(50), cancellationToken); + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqHandoffFailureTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqHandoffFailureTests.cs new file mode 100644 index 00000000..605e3ad2 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqHandoffFailureTests.cs @@ -0,0 +1,566 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Aspire.Hosting.Testing; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; +using Foundatio.Xunit; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +[Collection(nameof(RabbitMqTestCollection))] +public class RabbitMqHandoffFailureTests(AspireFixture fixture, ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_AfterRetryHandoffRecovery_PreservesIdentityWithoutRebroadcastAsync(bool publicationWasAccepted) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await using var healthy = new RabbitMQMessageBus(context.Options(healthy: true)); + var healthyIds = new ConcurrentQueue(); + await healthy.SubscribeAsync>(message => healthyIds.Enqueue(message.UniqueId!), token); + var options = context.Options(); + // Normal producer confirms remain off; the retry channel must still require them. + options.PublisherConfirmsEnabled = false; + await using var failed = new FaultedHandoffBus(options, publicationWasAccepted); + var deliveryIds = new ConcurrentQueue(); + int attempts = 0; + await failed.SubscribeAsync>(message => + { + deliveryIds.Enqueue(message.UniqueId!); + if (Interlocked.Increment(ref attempts) == 1) + throw new InvalidOperationException("Synthetic first attempt failure"); + }, token); + string id = Guid.NewGuid().ToString("N"); + // Act + await failed.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + await failed.FirstFailedAttempt.Task.WaitAsync(token); + await RabbitMqReliabilityTestContext.WaitAsync(() => failed.LastDeliveryError is not null, token); + // Assert + Assert.True(GetSubscriberConnection(failed).IsOpen); + failed.AllowRecovery(); + int expectedAttempts = publicationWasAccepted ? 3 : 2; + await RabbitMqReliabilityTestContext.WaitAsync(() => Volatile.Read(ref attempts) == expectedAttempts + && failed.ActiveDeliveryCount == 0 && healthyIds.Count == 1 && healthy.ActiveDeliveryCount == 0, token); + Assert.Equal(Enumerable.Repeat(id, expectedAttempts), deliveryIds.ToArray()); + Assert.Equal(new[] { id }, healthyIds.ToArray()); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + Assert.True(failed.IsSubscriptionReady); + _logger.LogInformation("Recovered accepted={PublicationWasAccepted}; attempts={ExpectedAttempts}; independent deliveries=1; stableId={MessageId}", + publicationWasAccepted, expectedAttempts, id); + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task SubscribeAsync_WithBrokerRejectedHandoff_ReusesHealthyTransportAsync(bool rabbitMq43, bool quorum, bool returned) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + string connectionString = rabbitMq43 ? fixture.MessagingPriority43ConnectionString! : fixture.MessagingConnectionString!; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(connectionString, Log, token); + await context.CreateDestinationAsync(token, bounded: true); + if (returned) + await context.Admin.QueueUnbindAsync(context.Destination, context.Dlx, "quarantine", cancellationToken: token); + else + await context.Admin.BasicPublishAsync(context.Dlx, "quarantine", true, + new BasicProperties { MessageId = "test-owned-filler" }, Encoding.UTF8.GetBytes("filler"), token); + var options = context.Options(quorum); + options.DeliveryLimit = 0; + int sockets = 0; + await using var bus = new RejectedHandoffBus(options, () => Volatile.Read(ref sockets)); + await bus.SubscribeAsync(_ => throw new InvalidOperationException("Synthetic terminal failure"), token); + var field = typeof(RabbitMQMessageBus).GetField("_factory", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + var factory = Assert.IsType(field.GetValue(bus)); + var socketFactory = factory.SocketFactory; + factory.SocketFactory = addressFamily => + { + Interlocked.Increment(ref sockets); + return socketFactory(addressFamily); + }; + + // Act + await bus.PublishAsync(new SimpleMessageA { Data = context.Topic }, new MessageOptions { UniqueId = context.Topic }, token); + var rejection = await bus.RepeatedRejection.Task.WaitAsync(token); + + // Assert + Assert.Equal(returned, rejection.IsReturn); + Assert.True(bus.SocketsAfterFirstRejection > 0); + Assert.Equal(bus.SocketsAfterFirstRejection, Volatile.Read(ref sockets)); + Assert.False(bus.IsSubscriptionReady); + if (returned) + await context.Admin.QueueBindAsync(context.Destination, context.Dlx, "quarantine", cancellationToken: token); + else + Assert.Equal("test-owned-filler", (await context.ReadAsync(context.Destination, token)).BasicProperties.MessageId); + Assert.Equal(context.Topic, (await context.ReadAsync(context.Destination, token)).BasicProperties.MessageId); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + Assert.Equal(bus.SocketsAfterFirstRejection, Volatile.Read(ref sockets)); + Assert.True(bus.IsSubscriptionReady); + Assert.Null(bus.LastDeliveryError); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_WithDifferentPublisherIdentity_TransfersOnlyToQuarantineAsync(bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await context.CreateDestinationAsync(token); + await context.Admin.QueueDeclareAsync(context.Healthy, true, false, false, cancellationToken: token); + await context.Admin.QueueBindAsync(context.Healthy, context.Dlx, "unintended", cancellationToken: token); + var originalUri = new Uri(fixture.MessagingConnectionString!); + using var management = fixture.App.CreateHttpClient("messaging", "management"); + management.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes(Uri.UnescapeDataString(originalUri.UserInfo)))); + string user = context.Topic; + string password = Guid.NewGuid().ToString("N"); + using var created = await management.PutAsJsonAsync($"api/users/{user}", new { password, tags = "" }, token); + created.EnsureSuccessStatusCode(); + try + { + using var permissions = await management.PutAsJsonAsync($"api/permissions/%2F/{user}", new + { + configure = $"^{context.Topic}.*$", + read = $"^{context.Topic}.*$", + write = $"^(amq.default|{context.Topic}.*)$" + }, token); + permissions.EnsureSuccessStatusCode(); + var options = context.Options(quorum); + options.ConnectionString = new UriBuilder(originalUri) { UserName = user, Password = password }.Uri.AbsoluteUri; + options.DeliveryLimit = 0; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => throw new InvalidOperationException("Synthetic terminal failure"), token); + var properties = new BasicProperties + { + UserId = Uri.UnescapeDataString(originalUri.UserInfo.Split(':')[0]), + MessageId = "cross-account", + Persistent = true, + Headers = new Dictionary { ["CC"] = new object[] { "unintended" }, ["application-header"] = "keep" } + }; + + // Act + await context.Admin.BasicPublishAsync(context.Topic, String.Empty, true, properties, Encoding.UTF8.GetBytes("payload"), token); + var terminal = await context.ReadAsync(context.Destination, token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + + // Assert + Assert.Null(terminal.BasicProperties.UserId); + Assert.Equal("cross-account", terminal.BasicProperties.MessageId); + Assert.Equal("payload", Encoding.UTF8.GetString(terminal.Body.Span)); + Assert.DoesNotContain("CC", terminal.BasicProperties.Headers!); + Assert.Contains("application-header", terminal.BasicProperties.Headers!); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Healthy, token)).MessageCount); + Assert.True(bus.IsSubscriptionReady); + } + finally + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + using var deleted = await management.DeleteAsync($"api/users/{user}", cleanup.Token); + deleted.EnsureSuccessStatusCode(); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_WithFailedRetryHandoff_RetainsOriginalOnHealthySubscriberAsync(bool publicationWasAccepted) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + var options = context.Options(); + options.PublisherConfirmsEnabled = false; + await using var bus = new FaultedHandoffBus(options, publicationWasAccepted); + await bus.SubscribeAsync(_ => throw new InvalidOperationException("Synthetic handler failure"), token); + string id = Guid.NewGuid().ToString("N"); + // Act + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + await bus.FirstFailedAttempt.Task.WaitAsync(token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.LastDeliveryError is not null, token); + // Assert + Assert.True(GetSubscriberConnection(bus).IsOpen, "The original subscriber must still be capable of acknowledging."); + Assert.False(bus.IsSubscriptionReady); + Assert.Equal(publicationWasAccepted ? 1u : 0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + + await bus.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(10), token); + int expectedCopies = publicationWasAccepted ? 2 : 1; + for (int i = 0; i < expectedCopies; i++) + { + var retained = await context.ReadAsync(context.Source, token); + Assert.Equal(id, retained.BasicProperties.MessageId); + Assert.Contains(id, Encoding.UTF8.GetString(retained.Body.Span)); + } + Assert.Null(await context.Admin.BasicGetAsync(context.Source, false, token)); + _logger.LogInformation("Injected accepted={PublicationWasAccepted}; subscriber stayed open; retainedCopies={ExpectedCopies}; id={MessageId}", + publicationWasAccepted, expectedCopies, id); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task SubscribeAsync_WithFullClassicRetryQueue_ReportsBlockedUntilProgressAsync(bool rabbitMq43, bool required) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + string connectionString = rabbitMq43 ? fixture.MessagingPriority43ConnectionString! : fixture.MessagingConnectionString!; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(connectionString, Log, token); + var options = context.Options(); + options.RequireSuccessfulDispatch = required; + options.DeliveryLimit = -1; + options.Arguments!["x-max-length"] = 1L; + options.Overflow = QueueOverflowBehavior.RejectPublish; + await using var bus = new RabbitMQMessageBus(options); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var fail = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var retried = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var repair = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var healthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + int attempts = 0; + await bus.SubscribeAsync(async message => + { + if (message.Data == "healthy") + { + healthy.TrySetResult(); + return; + } + if (Interlocked.Increment(ref attempts) == 1) + { + entered.TrySetResult(); + await fail.Task.WaitAsync(token); + throw new InvalidOperationException("Synthetic retry after source fills"); + } + retried.TrySetResult(); + await repair.Task.WaitAsync(token); + }, token); + + try + { + // Act + await bus.PublishAsync(new SimpleMessageA { Data = "original" }, cancellationToken: token); + await entered.Task.WaitAsync(token); + await bus.PublishAsync(new SimpleMessageA { Data = "healthy" }, cancellationToken: token); + fail.TrySetResult(); + await retried.Task.WaitAsync(token); + + // Assert + Assert.False(bus.IsSubscriptionReady); + Assert.IsType(bus.LastDeliveryError); + Assert.False(healthy.Task.IsCompleted); + repair.TrySetResult(); + await healthy.Task.WaitAsync(token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + Assert.Equal(2, Volatile.Read(ref attempts)); + Assert.True(bus.IsSubscriptionReady); + Assert.Null(bus.LastDeliveryError); + } + finally + { + fail.TrySetResult(); + repair.TrySetResult(); + } + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task SubscribeAsync_WithFullClassicRetryQueue_RespectsRetryBudgetAndContinuesAsync(bool rabbitMq43, bool required, bool exhaust) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + string connectionString = rabbitMq43 ? fixture.MessagingPriority43ConnectionString! : fixture.MessagingConnectionString!; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(connectionString, Log, token); + await context.CreateDestinationAsync(token); + var options = context.Options(); + options.RequireSuccessfulDispatch = required; + options.DeliveryLimit = 2; + options.Arguments!["x-max-length"] = 1L; + options.Overflow = QueueOverflowBehavior.RejectPublish; + await using var bus = new RabbitMQMessageBus(options); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var fail = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var healthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var identities = new ConcurrentQueue(); + int attempts = 0; + await bus.SubscribeAsync>(async message => + { + if (message.Body.Data == "healthy") + { + healthy.TrySetResult(); + return; + } + identities.Enqueue(message.UniqueId!); + int attempt = Interlocked.Increment(ref attempts); + if (attempt == 1) + { + entered.TrySetResult(); + await fail.Task.WaitAsync(token); + } + if (exhaust || attempt == 1) + throw new InvalidOperationException("Synthetic retry after source fills"); + }, token); + + // Act + await bus.PublishAsync(new SimpleMessageA { Data = "original" }, new MessageOptions { UniqueId = "original" }, token); + await entered.Task.WaitAsync(token); + await bus.PublishAsync(new SimpleMessageA { Data = "healthy" }, cancellationToken: token); + fail.TrySetResult(); + await healthy.Task.WaitAsync(TimeSpan.FromSeconds(10), token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + + // Assert + int expectedAttempts = exhaust ? 3 : 2; + Assert.Equal(expectedAttempts, Volatile.Read(ref attempts)); + Assert.Equal(Enumerable.Repeat("original", expectedAttempts), identities.ToArray()); + if (exhaust) + Assert.Equal("original", (await context.ReadAsync(context.Destination, token)).BasicProperties.MessageId); + Assert.True(bus.IsSubscriptionReady); + Assert.Null(bus.LastDeliveryError); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_WithLeastPrivilegeClassicRetry_RequiresDefaultExchangeWritePermissionAsync(bool permitted) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + var originalUri = new Uri(fixture.MessagingConnectionString!); + using var management = fixture.App.CreateHttpClient("messaging", "management"); + management.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes(Uri.UnescapeDataString(originalUri.UserInfo)))); + string user = context.Topic; + string password = Guid.NewGuid().ToString("N"); + using var created = await management.PutAsJsonAsync($"api/users/{user}", new { password, tags = "" }, token); + created.EnsureSuccessStatusCode(); + try + { + string prefix = $"^{context.Topic}.*$"; + string retryWrite = $"^(amq\\.default|{context.Topic}.*)$"; + using var permissions = await management.PutAsJsonAsync($"api/permissions/%2F/{user}", new + { + configure = prefix, + read = prefix, + write = permitted ? retryWrite : prefix + }, token); + permissions.EnsureSuccessStatusCode(); + var options = context.Options(); + options.ConnectionString = new UriBuilder(originalUri) { UserName = user, Password = password }.Uri.AbsoluteUri; + options.RequireSuccessfulDispatch = false; + options.DeadLetterExchange = null; + int attempts = 0; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync>(message => + { + if (Interlocked.Increment(ref attempts) == 1) + throw new InvalidOperationException("Synthetic first attempt failure"); + received.TrySetResult(message.UniqueId!); + }, token); + + // Act + await bus.PublishAsync(new SimpleMessageA { Data = "retry" }, new MessageOptions { UniqueId = "retry" }, token); + if (!permitted) + { + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.LastDeliveryError is not null, token); + Assert.False(bus.IsSubscriptionReady); + Assert.Equal(1, Volatile.Read(ref attempts)); + Assert.False(received.Task.IsCompleted); + using var repaired = await management.PutAsJsonAsync($"api/permissions/%2F/{user}", new + { + configure = prefix, + read = prefix, + write = retryWrite + }, token); + repaired.EnsureSuccessStatusCode(); + } + string id = await received.Task.WaitAsync(token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + + // Assert + Assert.Equal("retry", id); + Assert.Equal(2, Volatile.Read(ref attempts)); + Assert.True(bus.IsSubscriptionReady); + Assert.Null(bus.LastDeliveryError); + } + finally + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + using var deleted = await management.DeleteAsync($"api/users/{user}", cleanup.Token); + deleted.EnsureSuccessStatusCode(); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task SubscribeAsync_WithMissingOrFullTerminalDestination_RetainsUntilRepairedAsync(bool full, bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + if (full) + { + await context.CreateDestinationAsync(token, bounded: true); + await context.Admin.BasicPublishAsync(context.Dlx, "quarantine", true, + new BasicProperties { MessageId = "test-owned-filler", Persistent = true }, Encoding.UTF8.GetBytes("filler"), token); + } + var options = context.Options(quorum); + options.DeliveryLimit = 0; + options.DeadLetterExchange = context.Dlx; + options.DeadLetterRoutingKey = "quarantine"; + int handlers = 0; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => + { + Interlocked.Increment(ref handlers); + throw new InvalidOperationException("Synthetic terminal failure"); + }, token); + string id = Guid.NewGuid().ToString("N"); + // Act + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.LastDeliveryError is not null, token); + // Assert + Assert.True(GetSubscriberConnection(bus).IsOpen); + Assert.False(bus.IsSubscriptionReady); + if (full) + { + var filler = await context.Admin.BasicGetAsync(context.Destination, true, token); + Assert.NotNull(filler); + Assert.Equal("test-owned-filler", filler.BasicProperties.MessageId); + } + else + { + await context.CreateDestinationAsync(token); + } + var transferred = await context.ReadAsync(context.Destination, token); + Assert.Equal(id, transferred.BasicProperties.MessageId); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + Assert.Equal(1, Volatile.Read(ref handlers)); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + Assert.True(bus.IsSubscriptionReady); + _logger.LogInformation("Terminal route repaired: full={DestinationWasFull}; handlerAttempts=1; confirmedId={MessageId}", + full, id); + } + + private static IConnection GetSubscriberConnection(RabbitMQMessageBus bus) + { + var field = typeof(RabbitMQMessageBus).GetField("_subscriberConnection", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsAssignableFrom(field.GetValue(bus)); + } + + private CancellationTokenSource CreateTimeout() + { + var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(40)); + return timeout; + } + + private sealed class FaultedHandoffBus(RabbitMQMessageBusOptions options, bool publicationWasAccepted) : RabbitMQMessageBus(options) + { + private int _allowRecovery; + private int _handoffAttempts; + internal TaskCompletionSource FirstFailedAttempt { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + internal void AllowRecovery() => Volatile.Write(ref _allowRecovery, 1); + + protected override async Task PublishHandoffAsync(string exchange, string routingKey, BasicProperties properties, + ReadOnlyMemory body, CancellationToken cancellationToken) + { + int attempt = Interlocked.Increment(ref _handoffAttempts); + if (Volatile.Read(ref _allowRecovery) == 0) + { + if (publicationWasAccepted && attempt == 1) + await base.PublishHandoffAsync(exchange, routingKey, properties, body, cancellationToken); + FirstFailedAttempt.TrySetResult(); + // For accepted=true, the real broker has already confirmed the copy. + // The caller observes failure instead, modeling an ambiguous handoff result. + throw new MessageBusException("Synthetic caller-observed handoff failure"); + } + await base.PublishHandoffAsync(exchange, routingKey, properties, body, cancellationToken); + } + } + + private sealed class RejectedHandoffBus(RabbitMQMessageBusOptions options, Func getSocketCount) : RabbitMQMessageBus(options) + { + private int _rejections; + internal int SocketsAfterFirstRejection { get; private set; } + internal TaskCompletionSource RepeatedRejection { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + protected override async Task PublishHandoffAsync(string exchange, string routingKey, BasicProperties properties, + ReadOnlyMemory body, CancellationToken cancellationToken) + { + try + { + await base.PublishHandoffAsync(exchange, routingKey, properties, body, cancellationToken); + } + catch (PublishException exception) + { + int rejections = Interlocked.Increment(ref _rejections); + if (rejections == 1) + SocketsAfterFirstRejection = getSocketCount(); + if (rejections >= 3) + RepeatedRejection.TrySetResult(exception); + throw; + } + } + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqLifecycleSafetyTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqLifecycleSafetyTests.cs new file mode 100644 index 00000000..868d5c41 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqLifecycleSafetyTests.cs @@ -0,0 +1,660 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.AsyncEx; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; +using Foundatio.Xunit; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using RabbitMQ.Client.Exceptions; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +[Collection(nameof(RabbitMqTestCollection))] +public class RabbitMqLifecycleSafetyTests(AspireFixture fixture, ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task DisposeAsync_WithUncooperativeHandler_RetainsDeliveryAndOwnedBodyAsync(bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await using var bus = new RabbitMQMessageBus(context.Options(quorum)); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lateBody = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + string id = Guid.NewGuid().ToString("N"); + try + { + await bus.SubscribeAsync(async message => + { + entered.TrySetResult(); + // Deliberately ignore cancellation to prove transport cleanup is not + // dependent on cooperative application code or pooled payload lifetime. + await release.Task; + lateBody.TrySetResult(Encoding.UTF8.GetString(message.Data.Span)); + }, token); + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + await entered.Task.WaitAsync(token); + // Act + await bus.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(10), token); + // Assert + Assert.False(bus.IsSubscriptionReady); + var retained = await context.ReadAsync(context.Source, token); + Assert.Equal(id, retained.BasicProperties.MessageId); + release.TrySetResult(); + Assert.Contains(id, await lateBody.Task.WaitAsync(token)); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, false, true)] + [InlineData(false, true, false)] + [InlineData(false, true, true)] + [InlineData(true, false, false)] + [InlineData(true, false, true)] + [InlineData(true, true, false)] + [InlineData(true, true, true)] + public async Task SubscribeAsync_AfterChannelReplacement_CancelsOriginalHandlerAsync(bool rabbitMq43, bool quorum, bool required) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + string connectionString = rabbitMq43 ? fixture.MessagingPriority43ConnectionString! : fixture.MessagingConnectionString!; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(connectionString, Log, token); + var options = context.Options(quorum); + options.RequireSuccessfulDispatch = required; + await using var bus = new RabbitMQMessageBus(options); + var first = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var replacement = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + int attempts = 0; + await bus.SubscribeAsync(async (_, handlerToken) => + { + if (Interlocked.Increment(ref attempts) == 1) + { + first.TrySetResult(handlerToken); + await Task.Delay(Timeout.InfiniteTimeSpan, handlerToken); + } + else + { + replacement.TrySetResult(handlerToken); + await release.Task.WaitAsync(handlerToken); + } + }, token); + await bus.PublishAsync(new SimpleMessageA { Data = context.Topic }, cancellationToken: token); + var originalToken = await first.Task.WaitAsync(token); + var channel = GetField(bus, "_subscriberChannel"); + + try + { + // Act + await channel.CloseAsync(cancellationToken: token); + var replacementToken = await replacement.Task.WaitAsync(token); + + // Assert + Assert.True(originalToken.IsCancellationRequested); + Assert.False(replacementToken.IsCancellationRequested); + Assert.False(token.IsCancellationRequested); + release.TrySetResult(); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + Assert.Equal(2, Volatile.Read(ref attempts)); + Assert.True(bus.IsSubscriptionReady); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task SubscribeAsync_AfterChannelReplacement_PreservesBestEffortHandlerAsync(bool rabbitMq43, bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + string connectionString = rabbitMq43 ? fixture.MessagingPriority43ConnectionString! : fixture.MessagingConnectionString!; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(connectionString, Log, token); + var options = context.Options(quorum); + options.AcknowledgementStrategy = AcknowledgementStrategy.FireAndForget; + options.RequireSuccessfulDispatch = false; + await using var bus = new RabbitMQMessageBus(options); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + int attempts = 0; + await bus.SubscribeAsync(async (_, handlerToken) => + { + Interlocked.Increment(ref attempts); + entered.TrySetResult(handlerToken); + await release.Task.WaitAsync(handlerToken); + completed.TrySetResult(); + }, token); + await bus.PublishAsync(new SimpleMessageA { Data = context.Topic }, cancellationToken: token); + var handlerToken = await entered.Task.WaitAsync(token); + var channel = GetField(bus, "_subscriberChannel"); + + try + { + // Act + await channel.CloseAsync(cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.IsSubscriptionReady + && !ReferenceEquals(channel, GetField(bus, "_subscriberChannel")), token); + + // Assert + Assert.False(handlerToken.IsCancellationRequested); + release.TrySetResult(); + await completed.Task.WaitAsync(token); + Assert.Equal(1, Volatile.Read(ref attempts)); + } + finally + { + release.TrySetResult(); + } + } + + [Theory] + [InlineData(AcknowledgementStrategy.FireAndForget)] + [InlineData(AcknowledgementStrategy.Automatic)] + public async Task SubscribeAsync_AfterLastSubscriptionCancellation_StopsConsumerAndResubscribesAsync( + AcknowledgementStrategy acknowledgementStrategy) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + using var subscription = CancellationTokenSource.CreateLinkedTokenSource(token); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + var options = context.Options(); + options.AcknowledgementStrategy = acknowledgementStrategy; + options.RequireSuccessfulDispatch = false; + await using var bus = new RabbitMQMessageBus(options); + int cancelledCalls = 0; + await bus.SubscribeAsync(_ => Interlocked.Increment(ref cancelledCalls), subscription.Token); + + // Act + await subscription.CancelAsync(); + while ((await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).ConsumerCount != 0) + await Task.Delay(TimeSpan.FromMilliseconds(50), token); + // Assert + Assert.False(bus.IsSubscriptionReady); + string id = Guid.NewGuid().ToString("N"); + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + Assert.Equal(1u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await bus.SubscribeAsync(message => received.TrySetResult(message.Data!), token); + Assert.Equal(id, await received.Task.WaitAsync(token)); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + Assert.Equal(0, Volatile.Read(ref cancelledCalls)); + Assert.True(bus.IsSubscriptionReady); + } + + [Fact] + public async Task SubscribeAsync_AfterQueueDeletion_RecreatesConsumerForNewWorkAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await using var bus = new RabbitMQMessageBus(context.Options()); + var receipts = new ConcurrentDictionary(); + await bus.SubscribeAsync(message => receipts.TryAdd(message.Data!, 0), token); + await bus.PublishAsync(new SimpleMessageA { Data = "before" }, cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => receipts.ContainsKey("before") && bus.ActiveDeliveryCount == 0, token); + var oldConsumer = GetField(bus, "_consumer"); + // Act + await context.Admin.QueueDeleteAsync(context.Source, cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.IsSubscriptionReady + && !ReferenceEquals(oldConsumer, GetField(bus, "_consumer")), token); + await bus.PublishAsync(new SimpleMessageA { Data = "after" }, cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => receipts.ContainsKey("after") && bus.ActiveDeliveryCount == 0, token); + // Assert + Assert.Equal(2, receipts.Count); + // Administrative deletion destroys queued work. Only post-recreation progress is claimed. + } + + [Theory] + [InlineData(AcknowledgementStrategy.FireAndForget)] + [InlineData(AcknowledgementStrategy.Automatic)] + public async Task SubscribeAsync_DuringConsumerCleanup_WaitsForReplacementConsumerAsync(AcknowledgementStrategy strategy) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + using var first = CancellationTokenSource.CreateLinkedTokenSource(token); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + var options = context.Options(); + options.RequireSuccessfulDispatch = false; + options.AcknowledgementStrategy = strategy; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => { }, first.Token); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task subscribing; + + // Act + using (await GetField(bus, "_subscriberLock").LockAsync(token)) + { + await first.CancelAsync(); + subscribing = bus.SubscribeAsync(message => received.TrySetResult(message.Data!), token); + try + { + Assert.False(subscribing.IsCompleted, "Subscription must wait for in-progress consumer cleanup."); + } + finally + { + var clear = typeof(RabbitMQMessageBus).GetMethod("ClearSubscriberChannelAsync", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(clear); + await Assert.IsAssignableFrom(clear.Invoke(bus, null)); + } + } + await subscribing.WaitAsync(token); + await bus.PublishAsync(new SimpleMessageA { Data = "replacement" }, cancellationToken: token); + + // Assert + Assert.Equal("replacement", await received.Task.WaitAsync(token)); + Assert.True(bus.IsSubscriptionReady); + Assert.Equal(1u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).ConsumerCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_RepeatedStartup_KeepsRegisteredConsumerAsync(bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + + for (int iteration = 0; iteration < 50; iteration++) + { + await using var bus = new RabbitMQMessageBus(context.Options(quorum)); + + // Act + await bus.SubscribeAsync(_ => { }, token); + var consumer = GetField(bus, "_consumer"); + bool ready = bus.IsSubscriptionReady; + await bus.SubscribeAsync(_ => { }, token); + + // Assert + Assert.True(ready); + Assert.Same(consumer, GetField(bus, "_consumer")); + Assert.True(bus.IsSubscriptionReady); + } + } + + [Fact] + public async Task SubscribeAsync_WhenOnlyCallerCancelsDuringSetup_DoesNotLeaveOrphanConsumerAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + using var subscription = CancellationTokenSource.CreateLinkedTokenSource(token); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + var options = context.Options(); + await context.Admin.QueueDeclareAsync(context.Source, true, false, false, options.Arguments, cancellationToken: token); + await using var bus = new GatedInitializationBus(options); + Task subscribing = bus.SubscribeAsync(_ => { }, subscription.Token); + try + { + await bus.InitializationStarted.Task.WaitAsync(token); + // Act + await subscription.CancelAsync(); + bus.ReleaseInitialization(); + await Assert.ThrowsAnyAsync(() => subscribing.WaitAsync(token)); + + while ((await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).ConsumerCount != 0) + await Task.Delay(TimeSpan.FromMilliseconds(50), token); + // Assert + Assert.False(bus.IsSubscriptionReady); + } + finally + { + bus.ReleaseInitialization(); + await Record.ExceptionAsync(async () => await subscribing.WaitAsync(token)); + } + } + + [Fact] + public async Task SubscribeAsync_WithCancelledSetupCaller_PreservesAnotherPendingSubscriptionAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + using var cancelledSubscription = CancellationTokenSource.CreateLinkedTokenSource(token); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await using var bus = new GatedInitializationBus(context.Options()); + int cancelledCalls = 0; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + Task first = bus.SubscribeAsync(_ => Interlocked.Increment(ref cancelledCalls), cancelledSubscription.Token); + Task? second = null; + try + { + await bus.InitializationStarted.Task.WaitAsync(token); + second = bus.SubscribeAsync(message => received.TrySetResult(message.Data!), token); + // Act + await cancelledSubscription.CancelAsync(); + + await Assert.ThrowsAnyAsync(() => first.WaitAsync(TimeSpan.FromSeconds(5), token)); + // Assert + Assert.False(second.IsCompleted); + bus.ReleaseInitialization(); + await second.WaitAsync(token); + string id = Guid.NewGuid().ToString("N"); + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + Assert.Equal(id, await received.Task.WaitAsync(token)); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + Assert.Equal(0, Volatile.Read(ref cancelledCalls)); + Assert.True(bus.IsSubscriptionReady); + Assert.Equal(1u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).ConsumerCount); + } + finally + { + bus.ReleaseInitialization(); + await Record.ExceptionAsync(async () => await first.WaitAsync(token)); + if (second is not null) + await Record.ExceptionAsync(async () => await second.WaitAsync(token)); + } + } + + [Fact] + public async Task SubscribeAsync_WithFailedInitialization_RemovesRegistrationAndAllowsRetryAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await context.Admin.QueueDeclareAsync(context.Source, true, false, false, + new Dictionary { ["x-queue-type"] = "classic" }, cancellationToken: token); + await using var bus = new RabbitMQMessageBus(context.Options(quorum: true)); + int failedRegistrationCalls = 0; + // Act + var error = await Assert.ThrowsAnyAsync(() => bus.SubscribeAsync( + _ => Interlocked.Increment(ref failedRegistrationCalls), token)); + // Assert + Assert.Equal((ushort)406, error.ShutdownReason!.ReplyCode); + Assert.False(bus.IsSubscriptionReady); + Assert.NotNull(bus.LastSubscriptionError); + + await context.Admin.QueueDeleteAsync(context.Source, cancellationToken: token); + int correctCalls = 0; + await bus.SubscribeAsync(_ => Interlocked.Increment(ref correctCalls), token); + await bus.PublishAsync(new SimpleMessageA { Data = context.Topic }, cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => Volatile.Read(ref correctCalls) == 1 && bus.ActiveDeliveryCount == 0, token); + Assert.Equal(0, Volatile.Read(ref failedRegistrationCalls)); + Assert.True(bus.IsSubscriptionReady); + Assert.Null(bus.LastSubscriptionError); + } + + [Theory] + [InlineData(AcknowledgementStrategy.FireAndForget, false, false)] + [InlineData(AcknowledgementStrategy.Automatic, false, false)] + [InlineData(AcknowledgementStrategy.Automatic, true, false)] + [InlineData(AcknowledgementStrategy.Automatic, true, true)] + public async Task SubscribeAsync_WithHandlerCancellation_UsesRetryPolicyAndContinuesAsync( + AcknowledgementStrategy acknowledgementStrategy, bool requireSuccessfulDispatch, bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + var options = context.Options(quorum); + options.AcknowledgementStrategy = acknowledgementStrategy; + options.RequireSuccessfulDispatch = requireSuccessfulDispatch; + await using var bus = new RabbitMQMessageBus(options); + string cancelledId = Guid.NewGuid().ToString("N"); + string nextId = Guid.NewGuid().ToString("N"); + int attempts = 0; + var completed = new ConcurrentDictionary(); + await bus.SubscribeAsync(message => + { + if (message.Data == cancelledId && Interlocked.Increment(ref attempts) == 1) + throw new OperationCanceledException("Synthetic handler-local cancellation"); + completed.AddOrUpdate(message.Data!, 1, (_, count) => count + 1); + }, token); + + // Act + await bus.PublishAsync(new SimpleMessageA { Data = cancelledId }, new MessageOptions { UniqueId = cancelledId }, token); + await bus.PublishAsync(new SimpleMessageA { Data = nextId }, new MessageOptions { UniqueId = nextId }, token); + bool retries = acknowledgementStrategy == AcknowledgementStrategy.Automatic; + await RabbitMqReliabilityTestContext.WaitAsync(() => completed.ContainsKey(nextId) + && (!retries || completed.ContainsKey(cancelledId)) && bus.ActiveDeliveryCount == 0, token); + + // Assert + Assert.False(token.IsCancellationRequested); + Assert.Equal(retries ? 2 : 1, Volatile.Read(ref attempts)); + Assert.Equal(retries ? 2 : 1, completed.Count); + Assert.Equal(1, completed[nextId]); + if (retries) + Assert.Equal(1, completed[cancelledId]); + Assert.Null(bus.LastDeliveryError); + Assert.True(bus.IsSubscriptionReady); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + } + + [Fact] + public async Task SubscribeAsync_WithLateHandlerCompletion_DoesNotSettleReplacementDeliveryAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await using var bus = new RabbitMQMessageBus(context.Options()); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var lateCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var receipts = new ConcurrentDictionary(); + int attempts = 0; + try + { + await bus.SubscribeAsync(async message => + { + if (Interlocked.Increment(ref attempts) == 1) + { + entered.TrySetResult(); + await release.Task; + lateCompleted.TrySetResult(); + return; + } + receipts.AddOrUpdate(message.Data!, 1, (_, count) => count + 1); + }, token); + await bus.PublishAsync(new SimpleMessageA { Data = "pending" }, cancellationToken: token); + await entered.Task.WaitAsync(token); + var connection = GetField(bus, "_subscriberConnection"); + var oldChannel = GetField(bus, "_subscriberChannel"); + // Act + await oldChannel.CloseAsync(200, "Synthetic channel replacement", cancellationToken: token).WaitAsync(token); + // Assert + Assert.True(connection.IsOpen); + await RabbitMqReliabilityTestContext.WaitAsync(() => receipts.ContainsKey("pending") && bus.IsSubscriptionReady && bus.ActiveDeliveryCount == 0, token); + var current = GetField(bus, "_subscriberChannel"); + Assert.NotSame(oldChannel, current); + release.TrySetResult(); + await lateCompleted.Task.WaitAsync(token); + await bus.PublishAsync(new SimpleMessageA { Data = "after" }, cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => receipts.ContainsKey("after") && bus.ActiveDeliveryCount == 0, token); + Assert.True(current.IsOpen); + Assert.Same(current, GetField(bus, "_subscriberChannel")); + Assert.Equal(1, receipts["pending"]); + Assert.Equal(1, receipts["after"]); + } + finally + { + release.TrySetResult(); + } + } + + [Fact] + public async Task SubscribeAsync_WithOneCancelledHandler_DispatchesToRemainingRequiredHandlerAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + using var subscription = CancellationTokenSource.CreateLinkedTokenSource(token); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await using var bus = new RabbitMQMessageBus(context.Options()); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await bus.SubscribeAsync(async (_, handlerToken) => + { + entered.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, handlerToken); + }, subscription.Token); + int completed = 0; + await bus.SubscribeAsync(_ => Interlocked.Increment(ref completed), token); + await bus.PublishAsync(new SimpleMessageA { Data = "first" }, cancellationToken: token); + await entered.Task.WaitAsync(token); + // Act + await subscription.CancelAsync(); + await RabbitMqReliabilityTestContext.WaitAsync(() => Volatile.Read(ref completed) == 1 && bus.ActiveDeliveryCount == 0, token); + await bus.PublishAsync(new SimpleMessageA { Data = "second" }, cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => Volatile.Read(ref completed) == 2 && bus.ActiveDeliveryCount == 0, token); + // Assert + Assert.True(bus.IsSubscriptionReady); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_WithRepeatedHandlerCancellation_ReachesTerminalDestinationAsync(bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await context.CreateDestinationAsync(token); + var options = context.Options(quorum); + options.DeadLetterExchange = context.Dlx; + options.DeadLetterRoutingKey = "quarantine"; + await using var bus = new RabbitMQMessageBus(options); + int attempts = 0; + await bus.SubscribeAsync(_ => + { + Interlocked.Increment(ref attempts); + throw new OperationCanceledException("Synthetic handler-local cancellation"); + }, token); + string id = Guid.NewGuid().ToString("N"); + + // Act + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + var terminal = await context.ReadAsync(context.Destination, token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + + // Assert + Assert.Equal(id, terminal.BasicProperties.MessageId); + Assert.Contains(id, Encoding.UTF8.GetString(terminal.Body.Span)); + Assert.Equal(2, Volatile.Read(ref attempts)); + Assert.True(bus.IsSubscriptionReady); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount); + } + + [Fact] + public async Task SubscribeAsync_WithSubscriptionOverride_UsesHookForSetupAndMaintenanceAsync() + { + // Arrange + using var timeout = CreateTimeout(); + await using var bus = new OverriddenSubscriptionBus(new RabbitMQMessageBusOptions { ConnectionString = "amqp://localhost" }); + + // Act + await bus.SubscribeAsync(_ => { }, timeout.Token); + + // Assert + Assert.Equal(1, bus.Calls); + Assert.True(await bus.MaintenanceCalled.Task.WaitAsync(timeout.Token)); + Assert.True(bus.Calls >= 2); + } + + private CancellationTokenSource CreateTimeout() + { + var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(40)); + return timeout; + } + + private static T GetField(RabbitMQMessageBus bus, string name) + { + var field = typeof(RabbitMQMessageBus).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsAssignableFrom(field.GetValue(bus)); + } + + private sealed class GatedInitializationBus(RabbitMQMessageBusOptions options) : RabbitMQMessageBus(options) + { + private readonly TaskCompletionSource _releaseInitialization = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal TaskCompletionSource InitializationStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + internal void ReleaseInitialization() => _releaseInitialization.TrySetResult(); + + protected override async Task EnsureTopicCreatedAsync(CancellationToken cancellationToken) + { + InitializationStarted.TrySetResult(); + await _releaseInitialization.Task.WaitAsync(cancellationToken); + await base.EnsureTopicCreatedAsync(cancellationToken); + } + } + + private sealed class OverriddenSubscriptionBus(RabbitMQMessageBusOptions options) : RabbitMQMessageBus(options) + { + private int _calls; + internal int Calls => Volatile.Read(ref _calls); + internal TaskCompletionSource MaintenanceCalled { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + protected override Task EnsureTopicCreatedAsync(CancellationToken cancellationToken) => + throw new InvalidOperationException("Subscription setup bypassed the override."); + + protected override Task EnsureTopicSubscriptionAsync(CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _calls) > 1) + MaintenanceCalled.TrySetResult(cancellationToken == DisposedCancellationToken); + else + Assert.Equal(DisposedCancellationToken, cancellationToken); + return Task.CompletedTask; + } + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqMessageBusOptionsTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqMessageBusOptionsTests.cs new file mode 100644 index 00000000..561994e7 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqMessageBusOptionsTests.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net.Security; +using System.Reflection; +using System.Threading.Tasks; +using Foundatio.AsyncEx; +using Foundatio.Messaging; +using Foundatio.Xunit; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +public class RabbitMqMessageBusOptionsTests(ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Theory] + [InlineData("amqp://localhost", false, 5672)] + [InlineData("amqp://localhost:5678", false, 5678)] + [InlineData("amqps://localhost", true, 5671)] + [InlineData("amqps://localhost:5679", true, 5679)] + public async Task Constructor_WithConnectionUri_PreservesEndpointTransportAsync(string connectionString, bool encrypted, int port) + { + // Act + await using var bus = new RabbitMQMessageBus(o => o.LoggerFactory(Log).ConnectionString(connectionString)); + + // Assert + var endpoint = Assert.Single(GetEndpoints(bus)); + Assert.Equal("localhost", endpoint.HostName); + Assert.Equal(port, endpoint.Port); + Assert.Equal(encrypted, endpoint.Ssl.Enabled); + Assert.Equal(SslPolicyErrors.None, endpoint.Ssl.AcceptablePolicyErrors); + if (encrypted) + Assert.Equal("localhost", endpoint.Ssl.ServerName); + } + + [Theory] + [InlineData("broker:0")] + [InlineData("broker:-1")] + [InlineData("broker:65536")] + [InlineData("broker:not-a-port")] + [InlineData("broker:")] + [InlineData(":5672")] + [InlineData("[::1")] + [InlineData("[::1]unexpected")] + [InlineData("[::1]:0")] + [InlineData("[]:5672")] + [InlineData(" ")] + public void Constructor_WithInvalidHost_RejectsConfigurationWithoutEchoingCredentials(string host) + { + // Arrange + var options = new RabbitMQMessageBusOptions + { + LoggerFactory = Log, + ConnectionString = "amqps://test-user:test-password@localhost", + Hosts = [host] + }; + + // Act + var exception = Record.Exception(() => new RabbitMQMessageBus(options)); + + // Assert + var argumentException = Assert.IsAssignableFrom(exception); + + Assert.DoesNotContain("test-password", argumentException.Message); + Assert.DoesNotContain("test-user", argumentException.Message); + } + + [Theory] + [InlineData("[::1]", "::1", 5671)] + [InlineData("[::1]:5680", "::1", 5680)] + [InlineData("::1", "::1", 5671)] + public async Task Constructor_WithIpv6Host_PreservesAddressAndTlsAsync(string host, string expectedHost, int expectedPort) + { + // Act + await using var bus = new RabbitMQMessageBus(o => o.LoggerFactory(Log).ConnectionString("amqps://localhost").Hosts(host)); + + // Assert + var endpoint = Assert.Single(GetEndpoints(bus)); + Assert.Equal(expectedHost, endpoint.HostName); + Assert.Equal(expectedPort, endpoint.Port); + Assert.True(endpoint.Ssl.Enabled); + Assert.Equal(expectedHost, endpoint.Ssl.ServerName); + } + + [Fact] + public async Task Constructor_WithIpv6Uri_UsesUnbracketedEndpointIdentityAsync() + { + // Act + await using var bus = new RabbitMQMessageBus(o => o.LoggerFactory(Log).ConnectionString("amqps://[::1]:5680")); + + // Assert + var endpoint = Assert.Single(GetEndpoints(bus)); + Assert.Equal("::1", endpoint.HostName); + Assert.Equal("::1", endpoint.Ssl.ServerName); + Assert.True(endpoint.Ssl.Enabled); + Assert.Equal(5680, endpoint.Port); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Constructor_WithReplacementHosts_PreservesTlsAndCredentialsAsync(bool useBuilder) + { + // Arrange + const string connectionString = "amqps://test-user:test-password@original.example:5679/test-vhost"; + string[] hosts = ["first.example", "second.example:5680", "FIRST.example", " "]; + // Act + await using var bus = useBuilder + ? new RabbitMQMessageBus(o => o.LoggerFactory(Log).ConnectionString(connectionString).Hosts(hosts)) + : new RabbitMQMessageBus(new RabbitMQMessageBusOptions { ConnectionString = connectionString, LoggerFactory = Log, Hosts = hosts }); + + // Assert + var endpoints = GetEndpoints(bus); + Assert.Equal(2, endpoints.Count); + Assert.Equal("first.example", endpoints[0].HostName); + Assert.Equal(5671, endpoints[0].Port); + Assert.Equal("second.example", endpoints[1].HostName); + Assert.Equal(5680, endpoints[1].Port); + Assert.NotSame(endpoints[0].Ssl, endpoints[1].Ssl); + Assert.All(endpoints, endpoint => + { + Assert.True(endpoint.Ssl.Enabled); + Assert.Equal(endpoint.HostName, endpoint.Ssl.ServerName); + Assert.Equal(SslPolicyErrors.None, endpoint.Ssl.AcceptablePolicyErrors); + }); + + var factory = GetField(bus, "_factory"); + Assert.Equal("test-user", factory.UserName); + Assert.Equal("test-password", factory.Password); + Assert.Equal("test-vhost", factory.VirtualHost); + Assert.All(endpoints, endpoint => Assert.Equal(factory.Ssl.Version, endpoint.Ssl.Version)); + } + + [Fact] + public void Constructor_WithRequiredDispatchAndFireAndForget_ThrowsArgumentException() + { + // Arrange + var options = new RabbitMQMessageBusOptions + { + ConnectionString = "amqp://localhost", + LoggerFactory = Log, + RequireSuccessfulDispatch = true + }; + + // Act + var exception = Record.Exception(() => new RabbitMQMessageBus(options)); + + // Assert + Assert.IsType(exception); + } + + [Fact] + public void Constructor_WithRequiredDispatchAndNoDestination_ThrowsArgumentException() + { + // Arrange + var options = new RabbitMQMessageBusOptions + { + ConnectionString = "amqp://localhost", + LoggerFactory = Log, + AcknowledgementStrategy = AcknowledgementStrategy.Automatic, + RequireSuccessfulDispatch = true + }; + + // Act + var exception = Assert.Throws(() => new RabbitMQMessageBus(options)); + + // Assert + Assert.Contains(nameof(options.DeadLetterExchange), exception.Message); + } + + [Fact] + public void Constructor_WithZeroUriPort_RejectsConfiguration() + { + // Arrange + var options = new RabbitMQMessageBusOptions { LoggerFactory = Log, ConnectionString = "amqps://localhost:0" }; + + // Act + var exception = Record.Exception(() => new RabbitMQMessageBus(options)); + + // Assert + Assert.IsAssignableFrom(exception); + } + + [Fact] + public async Task ConvertToMessage_WithNumericHeaders_UsesInvariantCultureAsync() + { + // Arrange + var originalCulture = CultureInfo.CurrentCulture; + await using var bus = new LifecycleOverrideBus(new RabbitMQMessageBusOptions + { + ConnectionString = "amqp://localhost", + LoggerFactory = Log + }); + var properties = new BasicProperties + { + Headers = new Dictionary { ["amount"] = 1234.5m } + }; + var envelope = new BasicDeliverEventArgs("consumer", 1, false, "source", "route", properties, new byte[] { 1 }); + try + { + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("fr-FR"); + + // Act + var message = bus.Convert(envelope); + + // Assert + Assert.Equal("1234.5", message.Properties["amount"]); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + } + } + + [Fact] + public void CopyHandoffProperties_WithPublisherIdentityAndRoutingHeaders_RemovesTransportControls() + { + // Arrange + var original = new BasicProperties + { + MessageId = "stable-id", + UserId = "original-publisher", + Persistent = true, + Headers = new Dictionary + { + ["CC"] = new object[] { "unintended-queue" }, + ["BCC"] = new object[] { "hidden-queue" }, + ["x-delay"] = 1000, + ["application-header"] = "preserved" + } + }; + var envelope = new BasicDeliverEventArgs("consumer", 1, false, "source", "route", original, new byte[] { 1 }); + + // Act + var properties = RabbitMQMessageConverter.CopyHandoffProperties(envelope); + + // Assert + Assert.Null(properties.UserId); + Assert.DoesNotContain("CC", properties.Headers!); + Assert.DoesNotContain("BCC", properties.Headers!); + Assert.DoesNotContain("x-delay", properties.Headers!); + Assert.Equal("stable-id", properties.MessageId); + Assert.Equal("preserved", properties.Headers!["application-header"]); + Assert.True(properties.Persistent); + Assert.Equal("original-publisher", original.UserId); + Assert.Contains("CC", original.Headers); + } + + [Theory] + [InlineData("_subscriberLock")] + [InlineData("_lock")] + [InlineData("_handoffLock")] + public async Task DisposeAsync_WithBusyTransportLock_ReturnsWithinTimeoutAsync(string field) + { + // Arrange + var bus = new RabbitMQMessageBus(new RabbitMQMessageBusOptions + { + ConnectionString = "amqp://localhost", + LoggerFactory = Log, + ShutdownTimeout = TimeSpan.FromMilliseconds(50) + }); + Task disposing; + using (await GetField(bus, field).LockAsync(TestCancellationToken)) + { + // Act + disposing = bus.DisposeAsync().AsTask(); + + // Assert + await disposing.WaitAsync(TimeSpan.FromSeconds(2), TestCancellationToken); + } + await disposing; + } + + [Fact] + public async Task DisposeAsync_WithLifecycleOverride_InvokesRemovalHookAsync() + { + // Arrange + var bus = new LifecycleOverrideBus(new RabbitMQMessageBusOptions + { + ConnectionString = "amqp://localhost", + LoggerFactory = Log + }); + + // Act + await bus.DisposeAsync(); + + // Assert + Assert.Equal(1, bus.Removals); + } + + private static List GetEndpoints(RabbitMQMessageBus bus) => GetField>(bus, "_endpoints"); + + private sealed class LifecycleOverrideBus(RabbitMQMessageBusOptions options) : RabbitMQMessageBus(options) + { + internal int Removals { get; private set; } + internal IMessage Convert(BasicDeliverEventArgs envelope) => ConvertToMessage(envelope); + + protected override Task RemoveTopicSubscriptionAsync() + { + Removals++; + return base.RemoveTopicSubscriptionAsync(); + } + } + + private static T GetField(RabbitMQMessageBus bus, string name) + { + var field = typeof(RabbitMQMessageBus).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsType(field.GetValue(bus)); + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqReliabilityTestContext.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqReliabilityTestContext.cs new file mode 100644 index 00000000..2935f9e8 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqReliabilityTestContext.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using RabbitMQ.Client.Exceptions; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +internal sealed class RabbitMqReliabilityTestContext : IAsyncDisposable +{ + private readonly IConnection _connection; + private readonly string _connectionString; + private readonly ILoggerFactory _loggerFactory; + internal IChannel Admin { get; } + internal string Topic { get; } = $"reliability-{Guid.NewGuid():N}"; + internal string Source => $"{Topic}-source"; + internal string Healthy => $"{Topic}-healthy"; + internal string Dlx => $"{Topic}-dlx"; + internal string Destination => $"{Topic}-destination"; + + private RabbitMqReliabilityTestContext(IConnection connection, IChannel admin, string connectionString, ILoggerFactory loggerFactory) + { + _connection = connection; + Admin = admin; + _connectionString = connectionString; + _loggerFactory = loggerFactory; + } + + internal static async Task CreateAsync(string connectionString, ILoggerFactory loggerFactory, CancellationToken token) + { + var factory = new ConnectionFactory { Uri = new Uri(connectionString), AutomaticRecoveryEnabled = false }; + var connection = await factory.CreateConnectionAsync(token); + try + { + var channel = await connection.CreateChannelAsync(new CreateChannelOptions(true, true), token); + return new RabbitMqReliabilityTestContext(connection, channel, connectionString, loggerFactory); + } + catch + { + await connection.DisposeAsync(); + throw; + } + } + + internal RabbitMQMessageBusOptions Options(bool quorum = false, bool healthy = false) => new() + { + ConnectionString = _connectionString, + LoggerFactory = _loggerFactory, + Topic = Topic, + SubscriptionQueueName = healthy ? Healthy : Source, + IsDurable = true, + IsSubscriptionQueueExclusive = false, + SubscriptionQueueAutoDelete = false, + AcknowledgementStrategy = AcknowledgementStrategy.Automatic, + RequireSuccessfulDispatch = true, + DeadLetterExchange = Dlx, + DeadLetterRoutingKey = "quarantine", + PublisherConfirmsEnabled = true, + PrefetchCount = 1, + DeliveryLimit = 1, + ShutdownTimeout = TimeSpan.FromSeconds(2), + NetworkRecoveryInterval = TimeSpan.FromSeconds(1), + Arguments = quorum + ? new Dictionary { ["x-queue-type"] = "quorum", ["x-delivery-limit"] = -1L } + : new Dictionary { ["x-queue-type"] = "classic" } + }; + + internal async Task CreateDestinationAsync(CancellationToken token, bool bounded = false) + { + await Admin.ExchangeDeclareAsync(Dlx, "direct", true, cancellationToken: token); + var arguments = bounded + ? new Dictionary { ["x-max-length"] = 1L, ["x-overflow"] = "reject-publish" } + : null; + await Admin.QueueDeclareAsync(Destination, true, false, false, arguments: arguments, cancellationToken: token); + await Admin.QueueBindAsync(Destination, Dlx, "quarantine", cancellationToken: token); + } + + internal async Task ReadAsync(string queue, CancellationToken token) + { + while (true) + { + var result = await Admin.BasicGetAsync(queue, false, token); + if (result is not null) + return result; + await Task.Delay(TimeSpan.FromMilliseconds(50), token); + } + } + + internal static async Task WaitAsync(Func condition, CancellationToken token) + { + while (!condition()) + await Task.Delay(TimeSpan.FromMilliseconds(50), token); + } + + public async ValueTask DisposeAsync() + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + try + { + foreach (string queue in new[] { Source, Healthy, Destination }) + { + await using var channel = await _connection.CreateChannelAsync(cancellationToken: timeout.Token); + try + { + await channel.QueueDeleteAsync(queue, cancellationToken: timeout.Token); + } + catch (OperationInterruptedException exception) when (exception.ShutdownReason?.ReplyCode == 404) + { + // A setup failure may occur before this test-owned resource exists. + } + } + foreach (string exchange in new[] { Topic, Dlx }) + { + await using var channel = await _connection.CreateChannelAsync(cancellationToken: timeout.Token); + try + { + await channel.ExchangeDeleteAsync(exchange, cancellationToken: timeout.Token); + } + catch (OperationInterruptedException exception) when (exception.ShutdownReason?.ReplyCode == 404) + { + } + } + } + finally + { + await Admin.DisposeAsync(); + await _connection.DisposeAsync(); + } + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqRequiredDeliveryTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqRequiredDeliveryTests.cs new file mode 100644 index 00000000..27372f73 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqRequiredDeliveryTests.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections.Concurrent; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; +using Foundatio.Xunit; +using RabbitMQ.Client; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +[Collection(nameof(RabbitMqTestCollection))] +public class RabbitMqRequiredDeliveryTests(AspireFixture fixture, ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Fact] + public async Task PublishAsync_WithBrokerDelay_SurvivesPublisherDisposalAsync() + { + Assert.SkipWhen(String.IsNullOrEmpty(fixture.MessagingDelayedConnectionString), "Delayed broker unavailable"); + + // Arrange + using var timeout = CreateTimeout(); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingDelayedConnectionString!, Log, timeout.Token); + var options = context.Options(); + options.RequireBrokerDelayedDelivery = true; + await using (var provision = new RabbitMQMessageBus(options)) + await provision.SubscribeAsync(_ => { }, timeout.Token); + string id = Guid.NewGuid().ToString("N"); + // Act + await using (var publisher = new RabbitMQMessageBus(options)) + await publisher.PublishAsync(new SimpleMessageA { Data = id }, + new MessageOptions { UniqueId = id, DeliveryDelay = TimeSpan.FromSeconds(2) }, timeout.Token); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var subscriber = new RabbitMQMessageBus(options); + await subscriber.SubscribeAsync(message => received.TrySetResult(message.Data!), timeout.Token); + // Assert + Assert.Equal(id, await received.Task.WaitAsync(timeout.Token)); + await RabbitMqReliabilityTestContext.WaitAsync(() => subscriber.ActiveDeliveryCount == 0, timeout.Token); + } + + [Fact] + public async Task PublishAsync_WithRequiredBrokerDelayAndNoPlugin_RejectsMemoryFallbackAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, timeout.Token); + var options = context.Options(); + options.RequireBrokerDelayedDelivery = true; + var receipts = new ConcurrentQueue(); + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(message => receipts.Enqueue(message.Data!), timeout.Token); + // Act + // Assert + await Assert.ThrowsAnyAsync(() => bus.PublishAsync(new SimpleMessageA { Data = "must-not-schedule" }, + new MessageOptions { DeliveryDelay = TimeSpan.FromMilliseconds(100) }, timeout.Token)); + await Task.Delay(TimeSpan.FromMilliseconds(300), timeout.Token); + Assert.Empty(receipts); + } + + [Fact] + public async Task PublishAsync_WithRequiredRoutingAndNoSubscription_ThrowsMessageBusExceptionAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, timeout.Token); + var options = context.Options(); + options.RequirePublishRouting = true; + options.PublisherConfirmsEnabled = false; + await using var bus = new RabbitMQMessageBus(options); + // Act + // Assert + await Assert.ThrowsAnyAsync(() => bus.PublishAsync( + new SimpleMessageA { Data = "unroutable" }, cancellationToken: timeout.Token)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_WithExhaustedDeliveryAndNoDestination_RetainsAcrossDisposalAsync(bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, timeout.Token); + var options = context.Options(quorum); + options.DeliveryLimit = 2; + int attempts = 0; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => + { + Interlocked.Increment(ref attempts); + throw new InvalidOperationException("Synthetic poisoned event"); + }, timeout.Token); + string id = Guid.NewGuid().ToString("N"); + // Act + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, timeout.Token); + await RabbitMqReliabilityTestContext.WaitAsync(() => Volatile.Read(ref attempts) >= 3 && bus.LastDeliveryError is not null, timeout.Token); + // Assert + Assert.False(bus.IsSubscriptionReady); + await bus.DisposeAsync(); + var retained = await context.ReadAsync(context.Source, timeout.Token); + Assert.Equal(id, retained.BasicProperties.MessageId); + Assert.Equal(3, Volatile.Read(ref attempts)); + } + + [Theory] + [InlineData("unknown")] + [InlineData("malformed")] + [InlineData("unmatched")] + public async Task SubscribeAsync_WithInvalidRequiredDelivery_TransfersToTerminalDestinationAsync(string failure) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, timeout.Token); + await context.CreateDestinationAsync(timeout.Token); + var options = context.Options(); + options.DeadLetterExchange = context.Dlx; + options.DeadLetterRoutingKey = "quarantine"; + int handlers = 0; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => Interlocked.Increment(ref handlers), timeout.Token); + string id = Guid.NewGuid().ToString("N"); + string type = failure switch + { + "unknown" => "Unknown.Required.Event, Unknown.Required.Assembly", + "unmatched" => typeof(SimpleMessageB).AssemblyQualifiedName!, + _ => typeof(SimpleMessageA).AssemblyQualifiedName! + }; + byte[] body = Encoding.UTF8.GetBytes(failure == "malformed" ? "{" : "{}"); + // Act + await context.Admin.BasicPublishAsync(context.Topic, String.Empty, true, + new BasicProperties { MessageId = id, Type = type, Persistent = true }, body, timeout.Token); + var dead = await context.ReadAsync(context.Destination, timeout.Token); + // Assert + Assert.Equal(id, dead.BasicProperties.MessageId); + Assert.Equal(body, dead.Body.ToArray()); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, timeout.Token); + Assert.Equal(0, Volatile.Read(ref handlers)); + } + + [Fact] + public async Task SubscribeAsync_WithUnboundTerminalDestination_RetainsUntilRepairedAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, timeout.Token); + await context.Admin.ExchangeDeclareAsync(context.Dlx, "direct", true, cancellationToken: timeout.Token); + var options = context.Options(); + options.DeliveryLimit = 0; + options.DeadLetterExchange = context.Dlx; + options.DeadLetterRoutingKey = "quarantine"; + int handlers = 0; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => + { + Interlocked.Increment(ref handlers); + throw new InvalidOperationException("Synthetic terminal failure"); + }, timeout.Token); + string id = Guid.NewGuid().ToString("N"); + // Act + await bus.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, timeout.Token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.LastDeliveryError is not null, timeout.Token); + // Assert + Assert.False(bus.IsSubscriptionReady); + await context.CreateDestinationAsync(timeout.Token); + var dead = await context.ReadAsync(context.Destination, timeout.Token); + Assert.Equal(id, dead.BasicProperties.MessageId); + // Receipt by the destination can precede confirmation and the original ACK. + // Complete the callback before graceful disposal; fault tests cover that gap separately. + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, timeout.Token); + Assert.Equal(1, Volatile.Read(ref handlers)); + await bus.DisposeAsync(); + Assert.Equal(0u, (await context.Admin.QueueDeclarePassiveAsync(context.Source, timeout.Token)).MessageCount); + } + + private CancellationTokenSource CreateTimeout() + { + var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(40)); + return timeout; + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqScalingTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqScalingTests.cs index af23d6d1..29330cd6 100644 --- a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqScalingTests.cs +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqScalingTests.cs @@ -153,6 +153,7 @@ await publisher.PublishAsync(new SimpleMessageA { Data = $"prefetch-{i}" }, Assert.True(deliveredWhileBlocked <= prefetchCount, $"Expected at most {prefetchCount} messages delivered while consumer is blocked, but got {deliveredWhileBlocked}"); + Assert.True(deliveredWhileBlocked > 0, "The test must observe a delivery before checking prefetch."); } finally { @@ -476,6 +477,15 @@ await publisher.PublishAsync(new SimpleMessageA { Data = msg }, _logger.LogInformation("Message loss rate: {LossRate:P2} (published={Pub}, received rolling={Recv})", lossRate, published.Count, receivedRolling); Assert.True(lossRate < 0.1, $"Message loss rate should be under 10% with quorum queues, was {lossRate:P2}"); + + // Reconcile confirmed publication IDs; duplicates cannot hide missing work. + string[] expected = published.Distinct().OrderBy(id => id).ToArray(); + using var drain = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + drain.CancelAfter(TimeSpan.FromSeconds(30)); + await RabbitMqReliabilityTestContext.WaitAsync(() => expected.All(received.Contains), drain.Token); + Assert.Empty(expected.Except(received)); + _logger.LogInformation("Confirmed publications reconciled: {Count}; duplicate deliveries: {Duplicates}", + expected.Length, received.Count - received.Distinct().Count()); } [Fact] @@ -582,4 +592,100 @@ await subscriber2.SubscribeAsync(msg => await subscriber1.DisposeAsync(); } } + + [Fact] + public async Task SubscribeAsync_AfterRawConsumerDisconnect_RecoversEveryUnacknowledgedMessage() + { + Assert.SkipWhen(!fixture.ChaosClusterAvailable, "Chaos cluster not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(60)); + var token = timeout.Token; + string topic = $"scaling-inflight-{Guid.NewGuid():N}"; + string queueName = $"{topic}-inflight"; + string connectionString = Chaos.GetConnectionString("chaos-1"); + string[] expected = Enumerable.Range(0, 3).Select(i => $"inflight-{i}").ToArray(); + var factory = new ConnectionFactory + { + Uri = new Uri(connectionString), + AutomaticRecoveryEnabled = false + }; + await using var adminConnection = await factory.CreateConnectionAsync(token); + await using var admin = await adminConnection.CreateChannelAsync(cancellationToken: token); + var arguments = new System.Collections.Generic.Dictionary + { + ["x-queue-type"] = "quorum", + ["x-delivery-limit"] = 5L + }; + + try + { + await admin.ExchangeDeclareAsync(topic, "fanout", durable: true, cancellationToken: token); + await admin.QueueDeclareAsync(queueName, durable: true, exclusive: false, autoDelete: false, + arguments: arguments, cancellationToken: token); + await admin.QueueBindAsync(queueName, topic, String.Empty, cancellationToken: token); + await using var publisher = new RabbitMQMessageBus(o => o + .ConnectionString(connectionString) + .Topic(topic) + .PublisherConfirmsEnabled(true) + .LoggerFactory(Log)); + foreach (string id in expected) + await publisher.PublishAsync(new SimpleMessageA { Data = id }, cancellationToken: token); + + // BasicGet with autoAck=false creates real outstanding deliveries without + // blocking a callback. Graceful disposal of a provider with a deliberately + // blocked handler waits for that handler and is not a disconnect injector. + // Act + await using (var firstConnection = await factory.CreateConnectionAsync(token)) + await using (var firstChannel = await firstConnection.CreateChannelAsync(cancellationToken: token)) + { + var deliveryTags = new System.Collections.Generic.HashSet(); + foreach (string _ in expected) + { + var delivery = await firstChannel.BasicGetAsync(queueName, autoAck: false, cancellationToken: token); + Assert.NotNull(delivery); + Assert.True(deliveryTags.Add(delivery.DeliveryTag)); + } + Assert.Equal(expected.Length, deliveryTags.Count); + var held = await firstChannel.QueueDeclarePassiveAsync(queueName, token); + Assert.Equal(0u, held.MessageCount); + _logger.LogInformation("Held {Count} unacknowledged deliveries before disconnect", deliveryTags.Count); + } + + // Closing the first consumer's connection returns its unacknowledged + // deliveries to the durable queue. Wait on broker state, not a fixed sleep. + while ((await admin.QueueDeclarePassiveAsync(queueName, token)).MessageCount < expected.Length) + await Task.Delay(TimeSpan.FromMilliseconds(100), token); + + var received = new ConcurrentDictionary(); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var subscriber = new RabbitMQMessageBus(o => o + .ConnectionString(connectionString) + .Topic(topic) + .SubscriptionQueueName(queueName) + .AcknowledgementStrategy(AcknowledgementStrategy.Automatic) + .PrefetchCount(5) + .UseQuorumQueues() + .DeliveryLimit(5) + .LoggerFactory(Log)); + await subscriber.SubscribeAsync(msg => + { + received.AddOrUpdate(msg.Data!, 1, (_, count) => count + 1); + if (expected.All(received.ContainsKey)) + completed.TrySetResult(); + }, token); + await completed.Task.WaitAsync(token); + // Assert + Assert.Equal(expected.OrderBy(id => id), received.Keys.OrderBy(id => id)); + _logger.LogInformation("Recovered all {Count} expected IDs; duplicate deliveries: {Duplicates}", + expected.Length, received.Values.Sum(count => count - 1)); + } + finally + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await admin.QueueDeleteAsync(queueName, cancellationToken: cleanup.Token); + await admin.ExchangeDeleteAsync(topic, cancellationToken: cleanup.Token); + } + } } diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqTerminalContractTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqTerminalContractTests.cs new file mode 100644 index 00000000..7bdca636 --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqTerminalContractTests.cs @@ -0,0 +1,294 @@ +using System; +using System.Collections.Generic; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Reflection; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Aspire.Hosting.Testing; +using Foundatio.Messaging; +using Foundatio.Tests.Messaging; +using Foundatio.Xunit; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +[Collection(nameof(RabbitMqTestCollection))] +public class RabbitMqTerminalContractTests(AspireFixture fixture, ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Fact] + public async Task BasicRejectAsync_WithQuorumLimitAndUnavailableDestination_RetainsDeadLetterTransferAsync() + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + // RabbitMQ 4.2.5's dead_letter_worker_publisher_confirm_timeout defaults + // to 180000 ms. Allow that real retry interval without changing the broker. + // See rabbitmq-server v4.2.5, deps/rabbit/Makefile and rabbit_fifo_dlx_worker.erl. + timeout.CancelAfter(TimeSpan.FromMinutes(4)); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await context.Admin.ExchangeDeclareAsync(context.Dlx, "direct", true, cancellationToken: token); + var options = context.Options(quorum: true); + options.Arguments!["x-delivery-limit"] = 0L; + options.DeliveryLimit = 0; + options.DeadLetterExchange = context.Dlx; + options.DeadLetterRoutingKey = "quarantine"; + options.DeadLetterStrategy = DeadLetterStrategy.AtLeastOnce; + options.Overflow = QueueOverflowBehavior.RejectPublish; + await using (var provision = new RabbitMQMessageBus(options)) + await provision.SubscribeAsync(_ => { }, token); + string id = Guid.NewGuid().ToString("N"); + await using var publisher = new RabbitMQMessageBus(options); + await publisher.PublishAsync(new SimpleMessageA { Data = id }, new MessageOptions { UniqueId = id }, token); + var original = await context.ReadAsync(context.Source, token); + Assert.Equal(id, original.BasicProperties.MessageId); + // Act + await context.Admin.BasicRejectAsync(original.DeliveryTag, requeue: true, token); + // There is no provider consumer. This exercises the broker's independent limit + // and at-least-once dead-letter worker, not the client-side terminal handoff. + while ((await context.Admin.QueueDeclarePassiveAsync(context.Source, token)).MessageCount != 0) + await Task.Delay(TimeSpan.FromMilliseconds(50), token); + await Task.Delay(TimeSpan.FromSeconds(2), token); + await context.CreateDestinationAsync(token); + _logger.LogInformation("Repaired broker terminal route; awaiting retained transfer {MessageId}", + id); + var dead = await context.ReadAsync(context.Destination, token); + // Assert + Assert.Equal(id, dead.BasicProperties.MessageId); + Assert.Equal(original.Body.ToArray(), dead.Body.ToArray()); + _logger.LogInformation("Broker at-least-once terminal transfer completed: {MessageId}", + id); + } + + [Fact] + public async Task PublishAsync_WithRequiredDelayAndExistingFanout_RejectsMemoryFallbackAsync() + { + Assert.SkipWhen(String.IsNullOrEmpty(fixture.MessagingDelayedConnectionString), "Delayed broker unavailable"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingDelayedConnectionString!, Log, token); + await context.Admin.ExchangeDeclareAsync(context.Topic, "fanout", true, cancellationToken: token); + var options = context.Options(); + options.RequireBrokerDelayedDelivery = true; + int received = 0; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => Interlocked.Increment(ref received), token); + await bus.PublishAsync(new SimpleMessageA { Data = "immediate" }, cancellationToken: token); + await RabbitMqReliabilityTestContext.WaitAsync(() => Volatile.Read(ref received) == 1 && bus.ActiveDeliveryCount == 0, token); + // Act + // Assert + await Assert.ThrowsAnyAsync(() => bus.PublishAsync(new SimpleMessageA { Data = "required-delay" }, + new MessageOptions { DeliveryDelay = TimeSpan.FromMilliseconds(100) }, token)); + await Task.Delay(TimeSpan.FromMilliseconds(300), token); + Assert.Equal(1, Volatile.Read(ref received)); + } + + [Fact] + public async Task PublishAsync_WithRequiredRoutingOnPluginBroker_ReportsReturnAndRecoversAsync() + { + Assert.SkipWhen(String.IsNullOrEmpty(fixture.MessagingDelayedConnectionString), "Delayed broker unavailable"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingDelayedConnectionString!, Log, token); + var options = context.Options(); + options.RequirePublishRouting = true; + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(message => received.TrySetResult(message.Data!), token); + var field = typeof(RabbitMQMessageBus).GetField("_publisherChannel", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + var channel = Assert.IsAssignableFrom(field.GetValue(bus)); + var returned = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + channel.BasicReturnAsync += (_, args) => + { + returned.TrySetResult(args.ReplyCode); + return Task.CompletedTask; + }; + // Act + await context.Admin.QueueUnbindAsync(context.Source, context.Topic, String.Empty, cancellationToken: token); + await Assert.ThrowsAnyAsync(() => bus.PublishAsync( + new SimpleMessageA { Data = "unroutable" }, cancellationToken: token)); + // Assert + Assert.Equal((ushort)312, await returned.Task.WaitAsync(token)); + Assert.True(channel.IsOpen); + Assert.False(received.Task.IsCompleted); + await context.Admin.QueueBindAsync(context.Source, context.Topic, String.Empty, cancellationToken: token); + await bus.PublishAsync(new SimpleMessageA { Data = "routed" }, cancellationToken: token); + Assert.Equal("routed", await received.Task.WaitAsync(token)); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_WithExhaustedDeliveryAndNoDestination_DiscardsAndContinuesAsync(bool quorum) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + var options = context.Options(quorum); + options.RequireSuccessfulDispatch = false; + options.DeadLetterExchange = null; + options.DeliveryLimit = 0; + int failures = 0; + var healthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(message => + { + if (message.Data == "poison") + { + Interlocked.Increment(ref failures); + throw new InvalidOperationException("Synthetic exhausted message"); + } + healthy.TrySetResult(); + }, token); + + // Act + await bus.PublishAsync(new SimpleMessageA { Data = "poison" }, cancellationToken: token); + await bus.PublishAsync(new SimpleMessageA { Data = "healthy" }, cancellationToken: token); + await healthy.Task.WaitAsync(TimeSpan.FromSeconds(10), token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + + // Assert + Assert.Equal(1, Volatile.Read(ref failures)); + Assert.True(bus.IsSubscriptionReady); + Assert.Null(bus.LastDeliveryError); + await bus.DisposeAsync(); + Assert.Null(await context.Admin.BasicGetAsync(context.Source, false, token)); + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task SubscribeAsync_WithExhaustedDelivery_UsesBrokerDeadLetteringAsync(bool quorum, bool policy) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await context.CreateDestinationAsync(token); + var options = context.Options(quorum); + options.RequireSuccessfulDispatch = false; + options.DeliveryLimit = 0; + var originalUri = new Uri(fixture.MessagingConnectionString!); + using var management = fixture.App.CreateHttpClient("messaging", "management"); + management.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes(Uri.UnescapeDataString(originalUri.UserInfo)))); + string policyPath = $"api/policies/%2F/{context.Topic}"; + try + { + if (policy) + { + options.DeadLetterExchange = null; + options.DeadLetterRoutingKey = null; + using var created = await management.PutAsJsonAsync(policyPath, new + { + pattern = $"^{context.Source}$", + definition = new Dictionary + { + ["dead-letter-exchange"] = context.Dlx, + ["dead-letter-routing-key"] = "quarantine" + }, + priority = 1 + }, token); + created.EnsureSuccessStatusCode(); + } + var healthy = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(message => + { + if (message.Data == "poison") + throw new InvalidOperationException("Synthetic exhausted message"); + healthy.TrySetResult(); + }, token); + string id = Guid.NewGuid().ToString("N"); + + // Act + await bus.PublishAsync(new SimpleMessageA { Data = "poison" }, new MessageOptions { UniqueId = id }, token); + await bus.PublishAsync(new SimpleMessageA { Data = "healthy" }, cancellationToken: token); + var dead = await context.ReadAsync(context.Destination, token); + await healthy.Task.WaitAsync(TimeSpan.FromSeconds(10), token); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + + // Assert + Assert.Equal(id, dead.BasicProperties.MessageId); + Assert.NotNull(dead.BasicProperties.Headers); + Assert.True(dead.BasicProperties.Headers.ContainsKey("x-death")); + Assert.True(bus.IsSubscriptionReady); + Assert.Null(bus.LastDeliveryError); + } + finally + { + if (policy) + { + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + using var deleted = await management.DeleteAsync(policyPath, cleanup.Token); + deleted.EnsureSuccessStatusCode(); + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SubscribeAsync_WithInvalidOrExceededRetryMetadata_DoesNotResetBudgetAsync(bool malformed) + { + Assert.SkipWhen(!fixture.IsAvailable, "RabbitMQ infrastructure not available"); + + // Arrange + using var timeout = CreateTimeout(); + var token = timeout.Token; + await using var context = await RabbitMqReliabilityTestContext.CreateAsync(fixture.MessagingConnectionString!, Log, token); + await context.CreateDestinationAsync(token); + var options = context.Options(); + options.DeadLetterExchange = context.Dlx; + options.DeadLetterRoutingKey = "quarantine"; + int calls = 0; + await using var bus = new RabbitMQMessageBus(options); + await bus.SubscribeAsync(_ => + { + Interlocked.Increment(ref calls); + throw new InvalidOperationException("Synthetic exhausted message"); + }, token); + string id = Guid.NewGuid().ToString("N"); + var properties = new BasicProperties + { + MessageId = id, + Persistent = true, + Type = typeof(SimpleMessageA).AssemblyQualifiedName, + Headers = new Dictionary { ["x-delivery-count"] = malformed ? "invalid" : (object)Int64.MaxValue } + }; + // Act + await context.Admin.BasicPublishAsync(context.Topic, String.Empty, true, properties, + JsonSerializer.SerializeToUtf8Bytes(new SimpleMessageA { Data = id }), token); + var dead = await context.ReadAsync(context.Destination, token); + // Assert + Assert.Equal(id, dead.BasicProperties.MessageId); + await RabbitMqReliabilityTestContext.WaitAsync(() => bus.ActiveDeliveryCount == 0, token); + Assert.Equal(1, Volatile.Read(ref calls)); + } + + private CancellationTokenSource CreateTimeout() + { + var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(90)); + return timeout; + } +} diff --git a/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqTlsRecoveryTests.cs b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqTlsRecoveryTests.cs new file mode 100644 index 00000000..04b4252c --- /dev/null +++ b/tests/Foundatio.RabbitMQ.Tests/Messaging/RabbitMqTlsRecoveryTests.cs @@ -0,0 +1,270 @@ +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Net; +using System.Net.Security; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Foundatio.Messaging; +using Foundatio.Xunit; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using Xunit; + +namespace Foundatio.RabbitMQ.Tests.Messaging; + +// The relays forward opaque bytes. TLS still terminates at the TLS-only broker; +// no certificate-validation callback or plaintext AMQP listener is introduced. +[Collection(nameof(RabbitMqTestCollection))] +public class RabbitMqTlsRecoveryTests(AspireFixture fixture, ITestOutputHelper output) : TestWithLoggingBase(output) +{ + [Theory] + [InlineData("_publisherConnection")] + [InlineData("_subscriberConnection")] + public async Task PublishAsync_AfterEstablishedTlsPathLoss_RecoversAndDrainsPendingWorkAsync(string targetRole) + { + Assert.SkipWhen(String.IsNullOrEmpty(fixture.TlsConnectionString), "RabbitMQ TLS infrastructure not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(90)); + var broker = GetBrokerUri(); + string topic = $"tls-recovery-{Guid.NewGuid():N}"; + var adminFactory = new ConnectionFactory { Uri = broker, AutomaticRecoveryEnabled = false }; + adminFactory.Ssl.AcceptablePolicyErrors = SslPolicyErrors.None; + await using var adminConnection = await adminFactory.CreateConnectionAsync(timeout.Token); + await using var admin = await adminConnection.CreateChannelAsync(cancellationToken: timeout.Token); + var controlOptions = CreateOptions(broker, topic); + controlOptions.IsDurable = true; + await using var control = new RabbitMQMessageBus(controlOptions); + // Own the initial listeners even if later setup fails before the fault-test + // cleanup block is entered. Restarted listeners are disposed in that block. + await using var firstLease = new RabbitMqTcpRelay(broker, IPAddress.Loopback); + await using var secondLease = new RabbitMqTcpRelay(broker, IPAddress.Loopback); + var first = firstLease; + var second = secondLease; + int firstPort = first.Port; + int secondPort = second.Port; + var options = CreateOptions(broker, topic); + options.Hosts = [$"127.0.0.1:{firstPort}", $"127.0.0.1:{secondPort}"]; + options.IsDurable = true; + options.IsSubscriptionQueueExclusive = false; + options.SubscriptionQueueAutoDelete = false; + options.RequestedHeartbeat = TimeSpan.FromSeconds(2); + options.NetworkRecoveryInterval = TimeSpan.FromSeconds(1); + var receipts = new ConcurrentDictionary>(); + var deliveries = new ConcurrentDictionary(); + var bus = new RabbitMQMessageBus(options); + + try + { + await bus.SubscribeAsync((message, _) => + { + deliveries.AddOrUpdate(message.Id, 1, (_, count) => count + 1); + if (receipts.TryGetValue(message.Id, out var completion)) + completion.TrySetResult(message.Id); + return Task.CompletedTask; + }, cancellationToken: timeout.Token).WaitAsync(timeout.Token); + + string warmup = AddExpected(receipts); + await bus.PublishAsync(new Probe { Id = warmup }, cancellationToken: timeout.Token).WaitAsync(timeout.Token); + Assert.Equal(warmup, await receipts[warmup].Task.WaitAsync(timeout.Token)); + var publisher = GetConnection(bus, "_publisherConnection"); + var subscriber = GetConnection(bus, "_subscriberConnection"); + var target = GetConnection(bus, targetRole); + AssertTls(publisher); + AssertTls(subscriber); + int interruptedPort = target.Endpoint.Port; + Assert.Contains(interruptedPort, new[] { firstPort, secondPort }); + int alternatePort = interruptedPort == firstPort ? secondPort : firstPort; + var publisherShutdown = ObserveShutdownAsync(publisher); + var subscriberShutdown = ObserveShutdownAsync(subscriber); + var publisherRecovery = ObserveRecoveryAsync(publisher); + var subscriberRecovery = ObserveRecoveryAsync(subscriber); + _logger.LogInformation("Established TLS sessions: publisher={PublisherPort}, subscriber={SubscriberPort}; target={TargetRole}, interrupted={InterruptedPort}, alternate={AlternatePort}", + publisher.Endpoint.Port, subscriber.Endpoint.Port, targetRole, interruptedPort, alternatePort); + + // Close both test-owned paths, including the observed active endpoint. + // Keeping both unavailable temporarily makes the pending-work assertion + // deterministic. The backend and its shared queue remain available. + // Act + await first.DisposeAsync(); + await second.DisposeAsync(); + await Task.WhenAll(publisherShutdown, subscriberShutdown).WaitAsync(timeout.Token); + // Assert + Assert.False(publisher.IsOpen); + Assert.False(subscriber.IsOpen); + // Client shutdown can precede the broker removing its consumer and requeueing deliveries. + while ((await admin.QueueDeclarePassiveAsync(options.SubscriptionQueueName, timeout.Token)).ConsumerCount != 0) + await Task.Delay(TimeSpan.FromMilliseconds(50), timeout.Token); + string pending = AddExpected(receipts); + await control.PublishAsync(new Probe { Id = pending }, cancellationToken: timeout.Token).WaitAsync(timeout.Token); + var queue = await admin.QueueDeclarePassiveAsync(options.SubscriptionQueueName, timeout.Token); + Assert.True(queue.MessageCount > 0, "The shared broker must retain pending work while both client paths are down."); + Assert.False(receipts[pending].Task.IsCompleted); + _logger.LogInformation("Both client paths down: retainedMessages={RetainedMessages}, pendingId={MessageId}, delivered=false", + queue.MessageCount, pending); + + // Restore only the other endpoint. Recovery cannot accidentally pass by + // reconnecting the target connection to its original endpoint. + if (alternatePort == firstPort) + first = new RabbitMqTcpRelay(broker, IPAddress.Loopback, firstPort); + else + second = new RabbitMqTcpRelay(broker, IPAddress.Loopback, secondPort); + await Task.WhenAll(publisherRecovery, subscriberRecovery).WaitAsync(timeout.Token); + Assert.Equal(alternatePort, target.Endpoint.Port); + Assert.Same(publisher, GetConnection(bus, "_publisherConnection")); + Assert.Same(subscriber, GetConnection(bus, "_subscriberConnection")); + AssertTls(publisher); + AssertTls(subscriber); + Assert.Equal(pending, await receipts[pending].Task.WaitAsync(timeout.Token)); + + string afterRecovery = AddExpected(receipts); + await bus.PublishAsync(new Probe { Id = afterRecovery }, cancellationToken: timeout.Token).WaitAsync(timeout.Token); + Assert.Equal(afterRecovery, await receipts[afterRecovery].Task.WaitAsync(timeout.Token)); + Assert.Equal(receipts.Keys.OrderBy(id => id), deliveries.Keys.OrderBy(id => id)); + _logger.LogInformation("Recovered TLS sessions: publisher={PublisherPort}, subscriber={SubscriberPort}; expected={ExpectedIds}; duplicate deliveries={DuplicateDeliveries}", + publisher.Endpoint.Port, subscriber.Endpoint.Port, String.Join(',', receipts.Keys), deliveries.Values.Sum(count => count - 1)); + } + finally + { + // Restore the interrupted listener before cleanup, including assertion + // failures. Never alter the backend broker or leave a fault active. + try + { + if (first.IsStopped) + first = new RabbitMqTcpRelay(broker, IPAddress.Loopback, firstPort); + if (second.IsStopped) + second = new RabbitMqTcpRelay(broker, IPAddress.Loopback, secondPort); + await bus.DisposeAsync(); + using var cleanup = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await admin.QueueDeleteAsync(options.SubscriptionQueueName, cancellationToken: cleanup.Token); + await admin.ExchangeDeleteAsync(topic, cancellationToken: cleanup.Token); + } + finally + { + await first.DisposeAsync(); + await second.DisposeAsync(); + } + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task PublishAsync_WithCustomTlsPort_DeliversThroughConfiguredEndpointAsync(bool replacementHost, bool ipv6) + { + Assert.SkipWhen(String.IsNullOrEmpty(fixture.TlsConnectionString), "RabbitMQ TLS infrastructure not available"); + + // Arrange + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(60)); + var broker = GetBrokerUri(); + var address = ipv6 ? IPAddress.IPv6Loopback : IPAddress.Loopback; + await using var relay = new RabbitMqTcpRelay(broker, address); + var endpoint = new UriBuilder(broker) { Host = address.ToString(), Port = relay.Port }.Uri; + string host = ipv6 ? $"[::1]:{relay.Port}" : $"127.0.0.1:{relay.Port}"; + var options = CreateOptions(endpoint, $"tls-port-{Guid.NewGuid():N}"); + if (replacementHost) + { + options.ConnectionString = new UriBuilder(broker) { Host = "unused.invalid" }.Uri.AbsoluteUri; + options.Hosts = [host]; + } + await using var bus = new RabbitMQMessageBus(options); + var received = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + string id = Guid.NewGuid().ToString("N"); + // Act + await bus.SubscribeAsync((message, _) => + { + received.TrySetResult(message.Id); + return Task.CompletedTask; + }, cancellationToken: timeout.Token).WaitAsync(timeout.Token); + await bus.PublishAsync(new Probe { Id = id }, cancellationToken: timeout.Token).WaitAsync(timeout.Token); + // Assert + Assert.Equal(id, await received.Task.WaitAsync(timeout.Token)); + foreach (string role in new[] { "_publisherConnection", "_subscriberConnection" }) + { + var connection = GetConnection(bus, role); + AssertTls(connection); + Assert.Equal(relay.Port, connection.Endpoint.Port); + Assert.Equal(address.ToString(), connection.Endpoint.Ssl.ServerName); + } + _logger.LogInformation("Verified TLS delivery: replacementHost={ReplacementHost}, address={Address}, port={Port}, id={MessageId}", + replacementHost, address, relay.Port, id); + } + + private static string AddExpected(ConcurrentDictionary> receipts) + { + string id = Guid.NewGuid().ToString("N"); + Assert.True(receipts.TryAdd(id, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously))); + return id; + } + + private static Task ObserveShutdownAsync(IConnection connection) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + connection.ConnectionShutdownAsync += (_, _) => + { + completion.TrySetResult(); + return Task.CompletedTask; + }; + return completion.Task; + } + + private static Task ObserveRecoveryAsync(IConnection connection) + { + var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + connection.RecoverySucceededAsync += (_, _) => + { + completion.TrySetResult(); + return Task.CompletedTask; + }; + return completion.Task; + } + + private RabbitMQMessageBusOptions CreateOptions(Uri uri, string topic) => new() + { + ConnectionString = uri.AbsoluteUri, + LoggerFactory = Log, + Topic = topic, + SubscriptionQueueName = $"{topic}-subscription", + IsDurable = false, + IsSubscriptionQueueExclusive = true, + SubscriptionQueueAutoDelete = true, + AcknowledgementStrategy = AcknowledgementStrategy.Automatic, + PublisherConfirmsEnabled = true, + PrefetchCount = 1 + }; + + private Uri GetBrokerUri() + { + string? value = fixture.TlsConnectionString; + Assert.True(Uri.TryCreate(value, UriKind.Absolute, out var uri), "The Aspire fixture must provide its TLS broker URI."); + Assert.Equal("amqps", uri!.Scheme); + Assert.True(uri.IsLoopback); + return uri; + } + + private static IConnection GetConnection(RabbitMQMessageBus bus, string name) + { + var field = typeof(RabbitMQMessageBus).GetField(name, BindingFlags.Instance | BindingFlags.NonPublic); + Assert.NotNull(field); + return Assert.IsAssignableFrom(field.GetValue(bus)); + } + + private static void AssertTls(IConnection connection) + { + Assert.True(connection.IsOpen); + Assert.Equal(new Version(4, 2, 5), RabbitMQMessageBus.ParseServerVersion(connection.ServerProperties)); + Assert.True(connection.Endpoint.Ssl.Enabled); + Assert.Equal(SslPolicyErrors.None, connection.Endpoint.Ssl.AcceptablePolicyErrors); + } + + public sealed class Probe + { + public string Id { get; set; } = String.Empty; + } +}