From 2e49b549f5ac0fb3d062d99de8a3c63e4aa81797 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Mon, 12 Oct 2026 13:08:08 +0100 Subject: [PATCH 01/22] Fix StringSet(Expiration, ValueCondition) doc link: set, not delex (#3272) The overload (and StringSetAsync via inheritdoc) pointed at the DELEX command page; it was copied from StringDelete. Also add the missing to match the sibling overloads. Fixes #3269 --- src/StackExchange.Redis/Interfaces/IDatabase.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.cs b/src/StackExchange.Redis/Interfaces/IDatabase.cs index 131abeb2a..822ee4676 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.cs @@ -3782,7 +3782,8 @@ IEnumerable SortedSetScan( /// The expiry to set. /// The condition to enforce. /// The flags to use for this operation. - /// See . + /// if the string was set, otherwise. + /// See . #pragma warning disable RS0027 // Public API with optional parameter(s) should have the most parameters amongst its public overloads bool StringSet(RedisKey key, RedisValue value, Expiration expiry = default, ValueCondition when = default, CommandFlags flags = CommandFlags.None); #pragma warning restore RS0027 From 3679bd0fe7dfa1c9ced3148183c44bdd9c26be91 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 13 Oct 2026 10:59:35 +0100 Subject: [PATCH 02/22] Tolerate refused CLUSTER SLOTS/NODES during topology updates (#3275) * Tolerate refused CLUSTER SLOTS/NODES during topology updates (#3274) Proxied deployments (Azure Managed Redis / Redis Enterprise) can identify as a cluster while refusing CLUSTER SLOTS ("ERR command is not allowed"; Enterprise exposes SLOTS only under the OSS cluster API). That refusal surfaced as a logged *error* from GetEndpointsFromClusterNodes at connect and on every reconfigure, including the periodic topology refresh. A refusal is a posture, not a fault: catch RedisServerException per task - SLOTS refused falls back to the NODES-only path (as the method's own comment always intended), NODES refused means no topology view at all, so return null; both are logged at Information (EventIds 128/129). Tests: ClusterCommandsRefusedUnitTests drives the in-process test server with SLOTS (and SLOTS+NODES) refused, asserting discovery still works and no error is logged; CapturingLogger promoted from ConnectFailureRefreshTests to a shared test helper. * Observe both topology tasks, so an abandoned one never faults unobserved SLOTS and NODES are issued as a pair; if one await throws anything other than a server refusal, the other task is abandoned mid-flight - and both ride the same connection, so the orphan is likely to fault too. Attach ObserveErrors at creation so it can never surface as an UnobservedTaskException in the host. (Pre-existing gap: the bare sequential awaits had the same shape.) --- .../ConnectionMultiplexer.cs | 36 ++++++++- src/StackExchange.Redis/LoggerExtensions.cs | 12 +++ .../ClusterCommandsRefusedUnitTests.cs | 75 +++++++++++++++++++ .../ConnectFailureRefreshTests.cs | 30 -------- .../Helpers/CapturingLogger.cs | 39 ++++++++++ 5 files changed, 158 insertions(+), 34 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs create mode 100644 tests/StackExchange.Redis.Tests/Helpers/CapturingLogger.cs diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index b2d38cdc1..970561773 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2157,13 +2157,41 @@ public EndPoint[] GetEndPoints(bool configuredOnly = false) => // node including those serving nothing. Asked as a pair for symmetry - trusting the topology // cached from autoconfigure here would mean acting on possibly-stale data while deliberately // re-reading the other half + // ObserveErrors: if either await below throws, the *other* task is abandoned mid-flight, + // and both ride the same connection - a faulted orphan must never surface as an + // UnobservedTaskException in the host var slotsTask = ExecuteAsyncImpl( - RedisServer.GetClusterSlotsMessage(CommandFlags.None), ResultProcessor.ClusterSlots, null, server); + RedisServer.GetClusterSlotsMessage(CommandFlags.None), ResultProcessor.ClusterSlots, null, server).ObserveErrors(); var nodesTask = ExecuteAsyncImpl( - RedisServer.GetClusterNodesMessage(CommandFlags.None), ResultProcessor.ClusterNodes, null, server); + RedisServer.GetClusterNodesMessage(CommandFlags.None), ResultProcessor.ClusterNodes, null, server).ObserveErrors(); - var slots = await slotsTask.ForAwait(); - var clusterConfig = await nodesTask.ForAwait(); + ClusterSlotsResult? slots; + try + { + slots = await slotsTask.ForAwait(); + } + catch (RedisServerException ex) + { + // the server *refused* the command - a posture, not a fault: proxied deployments + // commonly answer NODES while blocking SLOTS (Redis Enterprise exposes SLOTS only + // under the OSS cluster API, for example; see #3274). Fall back to the NODES-only + // path, exactly as for a pre-4.0 server that has no SLOTS view to give + log?.LogInformationClusterSlotsRefused(new(server), ex.Message); + slots = null; + } + + ClusterConfiguration? clusterConfig; + try + { + clusterConfig = await nodesTask.ForAwait(); + } + catch (RedisServerException ex) + { + // likewise; a server refusing NODES as well exposes no topology at all, however it + // announced itself (a proxy's HELLO can say "cluster" while CLUSTER stays blocked) + log?.LogInformationClusterNodesRefused(new(server), ex.Message); + return null; + } if (clusterConfig is null) { return null; diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 6617f3df6..b8abcea8f 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -328,6 +328,18 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "Encountered error while updating cluster config: {ErrorMessage}")] internal static partial void LogErrorEncounteredErrorWhileUpdatingClusterConfig(this ILogger logger, Exception exception, string errorMessage); + [LoggerMessage( + Level = LogLevel.Information, + EventId = 128, + Message = "{Server}: CLUSTER SLOTS was refused ({ErrorMessage}); continuing with CLUSTER NODES alone")] + internal static partial void LogInformationClusterSlotsRefused(this ILogger logger, ServerEndPointLogValue server, string errorMessage); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 129, + Message = "{Server}: CLUSTER NODES was refused ({ErrorMessage}); no cluster topology is available")] + internal static partial void LogInformationClusterNodesRefused(this ILogger logger, ServerEndPointLogValue server, string errorMessage); + [LoggerMessage( Level = LogLevel.Information, EventId = 111, diff --git a/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs new file mode 100644 index 000000000..1f6e34de5 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs @@ -0,0 +1,75 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Proxied deployments can refuse some or all CLUSTER subcommands while still identifying as a cluster +/// (Redis Enterprise exposes CLUSTER SLOTS only under the OSS cluster API, and its HELLO/INFO +/// can say "cluster" while CLUSTER stays blocked entirely). A refusal is a posture, not a fault: the +/// topology pass must fall back to whatever view it can get, quietly - not log a server error on every +/// reconfigure. See #3274. +/// +[RunPerProtocol] +public class ClusterCommandsRefusedUnitTests(ITestOutputHelper log) +{ + private sealed class RefusingServer(ITestOutputHelper log, bool refuseNodes) : InProcessTestServer(log) + { + protected override TypedRedisValue ClusterSlots(RedisClient client, in RedisRequest request) + => TypedRedisValue.Error("ERR command is not allowed"); + + protected override TypedRedisValue ClusterNodes(RedisClient client, in RedisRequest request) + => refuseNodes ? TypedRedisValue.Error("ERR command is not allowed") : base.ClusterNodes(client, request); + } + + [Fact] + public async Task SlotsRefusedFallsBackToNodes() + { + using var server = new RefusingServer(log, refuseNodes: false) { ServerType = ServerType.Cluster }; + var a = server.DefaultEndPoint; + var b = server.AddEmptyNode(); + + var config = server.GetClientConfig(defaultOnly: true); // only connect to a initially + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + // discovery still happened, from the NODES view alone + var endpoints = conn.GetEndPoints(); + Assert.Contains(a, endpoints); + Assert.Contains(b, endpoints); + + log.WriteLine(logger.All); + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + Assert.NotEmpty(logger.Matching("CLUSTER SLOTS was refused")); + + await conn.GetDatabase().PingAsync(); + } + + [Fact] + public async Task AllClusterCommandsRefusedIsQuiet() + { + // the server still *identifies* as a cluster (HELLO "mode", INFO "redis_mode"), so the client + // interrogates it; both commands being refused must leave a working connection, not a logged error + using var server = new RefusingServer(log, refuseNodes: true) { ServerType = ServerType.Cluster }; + + var config = server.GetClientConfig(defaultOnly: true); + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + + log.WriteLine(logger.All); + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + Assert.NotEmpty(logger.Matching("CLUSTER NODES was refused")); + + await conn.GetDatabase().PingAsync(); + + // ...and a later explicit reconfigure stays quiet too + Assert.True(await conn.ReconfigureAsync("test")); + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + } +} diff --git a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs index 48ae8b973..0bc6beb2b 100644 --- a/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs +++ b/tests/StackExchange.Redis.Tests/ConnectFailureRefreshTests.cs @@ -145,34 +145,4 @@ public async Task TheReadIsRateLimitedRatherThanOncePerFailure() Assert.True(refreshes <= permitted, $"expected at most {permitted} rate-limited reads, but saw {refreshes}"); } } - - private sealed class CapturingLogger : ILoggerFactory, ILogger - { - private readonly List _messages = []; - - public ILogger CreateLogger(string categoryName) => this; - - public void AddProvider(ILoggerProvider provider) { } - - public IDisposable? BeginScope(TState state) where TState : notnull => null; - - public bool IsEnabled(LogLevel logLevel) => true; - - public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) - { - lock (_messages) _messages.Add(formatter(state, exception)); - } - - public List Matching(string fragment) - { - lock (_messages) return _messages.FindAll(x => x.Contains(fragment, StringComparison.Ordinal)); - } - - public string All - { - get { lock (_messages) return string.Join("\n", _messages); } - } - - public void Dispose() { } - } } diff --git a/tests/StackExchange.Redis.Tests/Helpers/CapturingLogger.cs b/tests/StackExchange.Redis.Tests/Helpers/CapturingLogger.cs new file mode 100644 index 000000000..eec70187e --- /dev/null +++ b/tests/StackExchange.Redis.Tests/Helpers/CapturingLogger.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace StackExchange.Redis.Tests; + +/// +/// An that records every formatted message, so a test can assert on what +/// was (or was not) logged. +/// +internal sealed class CapturingLogger : ILoggerFactory, ILogger +{ + private readonly List _messages = []; + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_messages) _messages.Add(formatter(state, exception)); + } + + public List Matching(string fragment) + { + lock (_messages) return _messages.FindAll(x => x.Contains(fragment, StringComparison.Ordinal)); + } + + public string All + { + get { lock (_messages) return string.Join("\n", _messages); } + } + + public void Dispose() { } +} From 209c46b02f564f55ff559452e26d447bb864a8f1 Mon Sep 17 00:00:00 2001 From: Diogo Martins <165835485+MDA2AV@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:07:22 +0100 Subject: [PATCH 03/22] Report a transport connection's failure the way a stream connection's is reported (#3273) --- .../PhysicalConnection.Transport.cs | 5 + src/StackExchange.Redis/PhysicalConnection.cs | 2 +- .../TransportCloseTests.cs | 153 ++++++++++++++++++ 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 tests/StackExchange.Redis.Tests/TransportCloseTests.cs diff --git a/src/StackExchange.Redis/PhysicalConnection.Transport.cs b/src/StackExchange.Redis/PhysicalConnection.Transport.cs index 9b7a53dc9..1e124bd83 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Transport.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Transport.cs @@ -20,10 +20,15 @@ internal sealed partial class PhysicalConnection // surface at all on this side. private DuplexTransport? _transport; + // Set when a transport is adopted and never cleared: the transport-mode counterpart of _ioStream, which + // RecordConnectionFailed checks before it reports a failure and fails what is awaiting a response. + private bool _transportAdopted; + internal bool HasTransport => _transport is not null; private void InitTransportOutput(DuplexTransport transport) { + _transportAdopted = true; _output = new TransportWriter(transport, OutputCancel); // Same reasoning as InitOutput: nothing awaits WriteComplete in production; observe it so diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index fc53e20e7..c960e4230 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -568,7 +568,7 @@ public void RecordConnectionFailed( var bridge = BridgeCouldBeNull; Message? nextMessage; - if (_ioStream is not null || isInitialConnect) // if *we* didn't burn the pipe: flag it + if (_ioStream is not null || _transportAdopted || isInitialConnect) // if *we* didn't burn the pipe: flag it { if (failureType == ConnectionFailureType.InternalFailure && innerException is not null) { diff --git a/tests/StackExchange.Redis.Tests/TransportCloseTests.cs b/tests/StackExchange.Redis.Tests/TransportCloseTests.cs new file mode 100644 index 000000000..8c70869a4 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/TransportCloseTests.cs @@ -0,0 +1,153 @@ +using System; +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite.Transports; +using StackExchange.Redis.Configuration; +using Xunit; + +namespace StackExchange.Redis.Tests; + +[RunPerProtocol] +public class TransportCloseTests(ITestOutputHelper output) +{ + [Fact] + public async Task PeerCloseFailsOutstandingCommandsAndRaisesConnectionFailed() + { + using var server = new InProcessTestServer(output); + var tunnel = new StreamTransportTunnel(server.Tunnel); + var config = server.GetClientConfig(withPubSub: false, defaultOnly: true); + config.Tunnel = tunnel; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var failed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + conn.ConnectionFailed += (_, e) => failed.TrySetResult(e); + + var db = conn.GetDatabase(); + await db.PingAsync(); + + // The next command is written to the transport but never reaches the server, so it is still + // awaiting its reply when the server closes the connection. + var transport = tunnel.Interactive!; + transport.DropWrites(); + var pending = db.PingAsync(); + await transport.Dropped.WaitAsync(TimeSpan.FromSeconds(5)); + transport.CloseAsIfByPeer(); + + var ex = await Assert.ThrowsAsync(() => pending); + Assert.Equal(ConnectionFailureType.SocketClosed, ex.FailureType); + + var args = await failed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.Equal(ConnectionFailureType.SocketClosed, args.FailureType); + } + + /// Connects through the in-process server's tunnel, but hands the client a + /// over that stream instead of the stream itself. + private sealed class StreamTransportTunnel(Tunnel inner) : Tunnel + { + public StreamTransport? Interactive { get; private set; } + + public override async ValueTask ConnectTransportAsync( + EndPoint endpoint, ConnectionType connectionType, TlsOptions tls, CancellationToken cancellationToken) + { + var stream = await inner.BeforeAuthenticateAsync(endpoint, connectionType, null, cancellationToken) + ?? throw new InvalidOperationException($"No in-process node at {endpoint}"); + var transport = new StreamTransport(stream); + if (connectionType == ConnectionType.Interactive) Interactive = transport; + return transport; + } + } + + /// A transport over a stream, with two levers for the test: drop what is written, and close the way + /// a transport does when its peer goes away - with no fault. + private sealed class StreamTransport(Stream stream) : DuplexTransport + { + private readonly CancellationTokenSource _peerClosed = new(); + private readonly TaskCompletionSource _dropped = new(TaskCreationOptions.RunContinuationsAsynchronously); + private byte[] _staged = new byte[1024]; + private int _stagedCount, _closed; + private volatile bool _dropWrites; + private TransportReceiver? _receiver; + + public Task Dropped => _dropped.Task; + + public void DropWrites() => _dropWrites = true; + + public void CloseAsIfByPeer() => _peerClosed.Cancel(); + + public override Memory GetMemory(int sizeHint = 0) => Reserve(sizeHint).AsMemory(_stagedCount); + + public override void Advance(int count) => _stagedCount += count; + + public override bool Flush() + { + if (Volatile.Read(ref _closed) != 0) + { + _stagedCount = 0; + return false; + } + + if (_stagedCount != 0) + { + if (_dropWrites) + { + _dropped.TrySetResult(true); + } + else + { + stream.Write(_staged, 0, _stagedCount); + stream.Flush(); + } + _stagedCount = 0; + } + return true; + } + + public override void Start(TransportReceiver receiver) + { + _receiver = receiver; + _ = Task.Run(ReceiveAsync); + } + + public override ValueTask DisposeAsync() + { + _peerClosed.Cancel(); + return default; + } + + private async Task ReceiveAsync() + { + var buffer = new byte[4096]; + Exception? fault = null; + try + { + while (true) + { + int read = await stream.ReadAsync(buffer, 0, buffer.Length, _peerClosed.Token); + if (read <= 0 || !_receiver!.OnReceived(new ReadOnlySpan(buffer, 0, read))) break; + _receiver.OnBatchEnd(); + } + } + catch (OperationCanceledException) when (_peerClosed.IsCancellationRequested) + { + // a clean close, as when the server ends the connection + } + catch (Exception ex) + { + fault = ex; + } + + Volatile.Write(ref _closed, 1); + stream.Dispose(); + _receiver!.OnClosed(fault); + } + + private byte[] Reserve(int sizeHint) + { + int needed = _stagedCount + Math.Max(sizeHint, 1); + if (needed > _staged.Length) Array.Resize(ref _staged, Math.Max(needed, _staged.Length * 2)); + return _staged; + } + } +} From d59ff9ef1677953619bfbc833f4cb2ec42f86ca8 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Tue, 13 Oct 2026 14:44:15 +0100 Subject: [PATCH 04/22] Lag-aware failback checks for geo-redundant failover (#3242) * Investigate Redis Enterprise lag-aware availability checks Confirms the premise: this is the client-side geographic failover feature family, and Lettuce, Jedis and redis-py have all shipped it. It feeds exactly the health checks our Availability/ namespace already has behind SER007. The abstractions line up almost one-to-one with redis-py's - probe, policy (all/any/majority), circuit breaker, per-member config - and two details make it land cleanly: HealthCheckProbe has no internal abstract members, so it is already subclassable from outside the assembly, and ConnectionGroupMember already carries a per-member HealthCheck override, which is what a per-cluster REST endpoint needs. Where we are better placed: HealthCheckResult has Inconclusive, which Lettuce's two-state HealthStatus lacks. Lettuce maps a failed REST call to UNHEALTHY, so a management-plane outage is indistinguishable from the data plane being down and can fail over a database that is serving fine. We can say "could not determine" instead. The awkward part is that src/ has no HTTP client and no JSON parser. JSON turns out to be avoidable if the bdb uid is configured rather than discovered, since the availability call is a status-code check. Also records the defaults disagreeing across clients: 100ms everywhere - server, docs, redis-py - but 5000ms in Lettuce. * Record what a Tunnel API for the management plane should look like mgravell offered to add one; the shape is what decides whether it helps. A member returning HttpMessageHandler would drag System.Net.Http into StackExchange.Redis's public API for the sake of one commercial deployment target - which is the thing keeping this in a separate package was meant to avoid, decided the expensive way. A stream-shaped member instead composes into SocketsHttpHandler.ConnectCallback, which takes exactly a ValueTask, so the Enterprise side gets a complete HTTP hijack and core gains no HTTP types at all. Tunnel already speaks that language via BeforeAuthenticateAsync and ConnectTransportAsync, and adding a virtual is binary-safe for subclasses. Two functional arguments for Tunnel over a separate knob, beyond testing: a CONNECT proxy configured for Redis almost certainly applies to the Enterprise REST API too, so co-locating makes that correct by construction; and the test harness already assigns Tunnel, so one assignment would spoof both planes. Caveats recorded: ConnectCallback is net5+, the existing members are EndPoint/ConnectionType-shaped where HTTP wants a Uri, and widening Tunnel's remit means HttpProxyTunnel and LoggingTunnel each need an opinion about whether they apply to the management plane. Also retires open question 5: RawConfig and ConfigurationOptions.Tunnel are both already public, so a probe reaches the tunnel from the context it already gets - no HealthCheckContext change needed. * Real clusters and the fault injector, and a correction I had written that inducing "reachable but stale" on demand is not something anyone can do in CI, so the fakes were the only route to the scenario that matters. That is wrong. Redis ships a fault injection service that the client suites drive over HTTP - POST /action with a type and parameters, poll the returned action_id - and Lettuce's ActiveActiveFailoverScenarioTest injects network_latency against a bdb_id for a bounded duration, which is exactly the mechanism for pushing replication lag past the tolerance and watching the lag-aware check flip while the plain check stays green. So the authoritative end-to-end test is real cluster plus fault injector, on a road Lettuce has already built. The fakes are demoted rather than removed: they still cover machines with no Enterprise cluster, they are far faster than an injector action that budgets minutes plus a stabilisation wait, and they cover what the injector does not obviously reach - specific error_code bodies, 401/403 credential expiry, slow-but-successful responses for probe timeouts. Also records what the real clusters are uniquely good for: everything in section 1 is read off documentation, and a short session settles whether extend_check=lag is actually supported, what failure bodies really contain, whether the Host header is required, what the bdbs listing really returns - which decides whether the JSON-avoidance argument holds - and what lag real geo links show, which is the empirical way to settle 100ms versus 5000ms. * We already have a fault-injector tier; it answers several of the open questions It is not merged - tests/StackExchange.Redis.FaultInjector.Tests lives on marc/maint-optin-server, open as #3191 (107 files, +13160) - which is presumably why I did not find it when I grepped a branch taken fresh from main. Two pieces of it bear directly on this. FaultInjectorClient wraps the same POST /action + GET /action/{id} contract go-redis, redis-py and node-redis converged on, and adds GetValidTriggersAsync, so we can ask a deployment which lag-inducing effects it supports instead of guessing at an action name. And ClusterRestClient is already a Redis Enterprise REST client on 9443 - Basic auth, System.Text.Json, GET /v1/bdbs?fields=uid,name - so the availability call is a method on an existing class, not a new project. It also settles by observation what I had listed as open and sourced only from documentation: the management certificate is self-signed per environment and pinned to a CA from the config directory, so the probe needs CA configuration and cannot rely on the ambient trust store; auth really is Basic; and the bdbs listing returns the requested fields, so discovery is easy - which weakens the avoid-JSON argument to "it keeps JSON out of src/", not "discovery is hard". Sequencing consequence recorded: an end-to-end test is downstream of #3191. And #3191's own summary is the best argument for validating before building - same vendor, same class of API: the specifications are prose, the payloads were never published, and nearly every starting assumption was wrong. * Add the plan, blocked on #3191 Step 0 is to wait for #3191 to merge and build from there, rather than writing a second Redis Enterprise REST client against the same cluster and reconciling the two later - its ClusterRestClient already does Basic auth, CA pinning and GET /v1/bdbs, and its FaultInjectorClient can be asked which lag-inducing effects a deployment supports. The one thing that can start earlier is the API validation, because it produces notes rather than code. Step 1 is that validation, before any implementation: a checklist of things currently taken from documentation, including the measurement that settles 100ms versus 5000ms, with captured bodies saved as fixtures so the stubbed tests replay real payloads. Step 2 deliberately does not decide where the code lives, but records that the dependency argument is weaker than it looked - System.Net.Http needs no package reference on any target we ship - so the question is judgement, not constraint. Also records that this branch becomes the feature branch when step 0 clears, and that the way to take the merge is to rebase onto main rather than merge main in, since #3191 squash-merges. * Step 0 is cleared; note that step 1 needs the machine with cluster access #3191 merged as dc915bbc, so this branch is rebased onto it - which is what step 0 asked for, and rebased rather than merged because #3191 squash-merged and a merge would have moved the merge base without bringing its ancestry. The fault-injector tier is present here now. Also recorded where the work can happen. Step 1 is validation against a real deployment plus the fault injector, and both live on the machine holding the environment directory; this box has no ~/aws and no env_output.json anywhere, so nothing in step 1 could be run or verified here. Worth writing down rather than rediscovering, since the environment being absent looks identical to the environment being offline. Steps 2 onward are ordinary code and are not tied to a machine. Noted too that the directory is the AWS multi-cluster template, so env_output.json nests its outputs under .clusters.value[0] - a shape FaultInjectorEnvironment already handles, so it needs no work, just no surprise. * Pin the cluster REST certificate per host rather than chaining to the config CA The 9443 certificate is self-signed by the cluster itself; the CA in the config directory only signs the RESP proxy certificates, so the chain build could never pass. Pin the leaf on first use, keyed by host so multi-cluster templates work. * Step 1 measured: the lag check is Active-Active only, and does what it says there Records the real-deployment results (findings section 10): off Active-Active the lag check fails unconditionally; on it, the stale member flips within seconds of sync pausing and the tolerance is honoured on a running link. Also the two integration problems found in Availability/, which mean src/ needs a change. * Correct the client defaults: Lettuce, Jedis and redis-py all default to 5000 ms Read from source: redis-py's 5000 contradicts its own docstring's 100. None of the three checks for Active-Active, and none has a third result state. * Measure what a probe can discover: not Active-Active over RESP, db_viewer suffices Active-Active membership has no RESP signal, so it comes from REST (fields=crdt) or configuration. db_viewer can call everything a probe needs, but unfiltered bdb/crdb reads expose database passwords even to it, so only filtered requests. The REST endpoint derives from the configured database hostname. * TLS reuse, the cross-client survey, and the local form routed by database host TrustIssuer(leaf) via TlsOptions validates the 9443 call unchanged. No other client activates lag checks automatically or checks crdt; go-redis routes the local form through each database host, which makes it truthful, and scopes the lag verdict to failback only. * The local form flips under paused sync exactly as the database form does Same verdict, same timing, stale member only, tolerance ignored while paused. Its failure code is bdb_endpoint_unavailable, shared with the wrong-node case, so the choice between forms is about call shape, not information. * Sketch the lag-aware API for review A failback-only health check per member rather than a new result value, a member-scoped probe, and a LagAware factory whose only required input is the management credential. Compared against Lettuce, Jedis, redis-py and go-redis. * API sketch: System.Text.Json decided; open questions compared with other clients Keep LagAware naming and failback (not fallback) terminology; rotating NetworkCredential callback plus a separate client-certificate option; and keep the liveness fallback, which no client has, because a dead peer probably makes the survivor's lag check fail. * Partition test: the lag check is blind for a minute, then fails on both sides Partitioning the clusters (tagged iptables rules, self-healing) showed no lag for ~60 s while sync had stopped, then tolerance-governed failure on both members; a tolerance of 0 is honoured. Supports the liveness fallback, and adds a caveat: failback in the first minute after a link breaks is not protected. * AllSuccess: a run of only inconclusive probes is not a pass The default policy reported Healthy when no probe failed, even if none succeeded. Harmless for liveness, where both count as connected, but wrong for any caller that distinguishes them. * Failback health checks: ask whether a member has caught up before moving to it (SER011) ConnectionGroupMember.FailbackHealthCheck (and a group default) is consulted only for members that are not active, and only Healthy makes them eligible. It never evicts the active member, and when no live member is eligible, selection falls back to liveness, since a partition leaves every region behind at once. HealthCheckProbe.Scope lets a probe run once per member rather than per endpoint. * Notes: core change implemented behind SER011 (SER007 is retired) * HealthCheck.LagAware: ask Redis Enterprise whether a member is available and caught up (SER011) A member-scoped probe of the database availability API. Only the management credentials are required: the REST endpoint derives from the configured host, the database is discovered with filtered requests only, the lag question is asked only of Active-Active databases, and no tolerance is sent unless set. Could-not-ask is Inconclusive, never Unhealthy. System.Text.Json on the down-level targets; custom certificate validation is unavailable on net461. * Notes: probe implemented; RawConfig is not public after all * LagAware: ask the cluster name, and never follow redirects Found against live clusters: non-master nodes answer the management routes (discovery, the crdt lookup) with a 307 to the master's internal address, so deriving the REST endpoint from the database host timed out for three of four members. Derive the cluster FQDN (strip redis-{port}.), which resolves to the master, and report a redirect as Inconclusive rather than following it. * Notes: management routes redirect to the master's internal address * Fault-injector scenario: refuse to fail back onto a stale Active-Active member Pauses sync into the preferred member through its cluster's REST API while writes go to the other, and asserts the group stays off it for 15s although it is alive, then fails back once it has caught up. The tier now reads every cluster's credentials, and shares its certificate pin with the probe. * Notes: tier-3 scenario test passing * LagAware: client certificates for the REST call LagAwareOptions.ClientCertificate, with SetUserPfxCertificate and (net6.0+) SetUserPemCertificate mirroring ConfigurationOptions and sharing its loaders. Wired into SocketsHttpHandler on .NET and HttpClientHandler down-level; not supported on net461. Every other client offers mTLS for this call. * Failback: a member chosen by the liveness fallback is provisional Other clients never select a member that fails its lag check; the nearest equivalent that still serves when every member is behind is to leave such a member for an eligible one as soon as one is available, whatever the weights, and to keep checking it so it becomes an ordinary active member once caught up. * Notes: fallback choice is provisional, as decided by client parity * Move the lag-aware log message to a free event id main now uses 124 and 125 (#3257); the guard test from #3266 rejects duplicates. * AnySuccess: a run of only inconclusive probes is not a failure AnySuccess reported Unhealthy whenever nothing succeeded, even when nothing failed either - so a probe that cannot tell (a key-write probe on a replica, say) marked its member unhealthy. Now Inconclusive, matching AllSuccess and MajoritySuccess. * Move the lag-aware log message to a free event id (again) main has since taken 128 as well. * Failback: a zero FailbackDelay clears a failure in the same clock tick; no ValueTuple UpdateState compared the last failure strictly before the cutoff. With FailbackDelay zero the cutoff is now, and on .NET Framework's ~15.6ms clock a failure and the passing check that clears it often share a tick, holding the member out for another interval. Compare inclusively. Also replace a named tuple in the failback-check pass: the library must not reference System.ValueTuple. * Ship lag-aware failback without an experiment Remove SER011: the attributes, the constant and NoWarn entry, the [SER011] API prefixes and docs/exp/SER011.md. It never shipped, so it is not a reserved id. Also restores main's using directives and encoding in the files that already existed there, which earlier edits had disturbed. * Notes: no experiment * Notes: drop the SER011 history --- Directory.Packages.props | 1 + docs/Failover.md | 88 +- notes/lag-aware/api-sketch.md | 260 ++++++ notes/lag-aware/findings.md | 759 ++++++++++++++++++ notes/lag-aware/plan.md | 204 +++++ notes/readme.md | 24 + .../Availability/HealthCheck.Execute.cs | 30 +- .../Availability/HealthCheck.LagAware.cs | 16 + .../Availability/HealthCheckProbe.LagAware.cs | 249 ++++++ .../Availability/HealthCheckProbe.cs | 5 + .../Availability/HealthCheckProbePolicy.cs | 12 +- .../Availability/HealthCheckProbeScope.cs | 19 + .../Availability/LagAwareOptions.cs | 135 ++++ .../Availability/MultiGroupMultiplexer.cs | 134 +++- .../Availability/MultiGroupOptions.cs | 24 +- .../ConfigurationOptions.cs | 21 +- src/StackExchange.Redis/LoggerExtensions.cs | 6 + .../PublicAPI/PublicAPI.Unshipped.txt | 34 + .../PublicAPI/net6.0/PublicAPI.Unshipped.txt | 1 + .../StackExchange.Redis.csproj | 4 + .../Environment/ClusterRestClient.cs | 125 ++- .../Environment/FaultInjectorEnvironment.cs | 43 +- .../Environment/FaultInjectorFixture.cs | 2 +- .../LagAwareFailbackScenarioTests.cs | 131 +++ .../README.md | 17 + .../HealthCheckPolicyUnitTests.cs | 2 + .../LagAwareProbeTests.cs | 401 +++++++++ .../FailbackHealthCheckTests.cs | 333 ++++++++ 28 files changed, 3024 insertions(+), 56 deletions(-) create mode 100644 notes/lag-aware/api-sketch.md create mode 100644 notes/lag-aware/findings.md create mode 100644 notes/lag-aware/plan.md create mode 100644 notes/readme.md create mode 100644 src/StackExchange.Redis/Availability/HealthCheck.LagAware.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckProbe.LagAware.cs create mode 100644 src/StackExchange.Redis/Availability/HealthCheckProbeScope.cs create mode 100644 src/StackExchange.Redis/Availability/LagAwareOptions.cs create mode 100644 tests/StackExchange.Redis.FaultInjector.Tests/LagAwareFailbackScenarioTests.cs create mode 100644 tests/StackExchange.Redis.Tests/LagAwareProbeTests.cs create mode 100644 tests/StackExchange.Redis.Tests/MultiGroupTests/FailbackHealthCheckTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index f822909d2..1009a1163 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,6 +8,7 @@ + diff --git a/docs/Failover.md b/docs/Failover.md index eb08eb4d4..58a4c43eb 100644 --- a/docs/Failover.md +++ b/docs/Failover.md @@ -83,6 +83,7 @@ Because the policies are immutable, there is no question of whether a change "ta | `RetryPolicy` | `MultiGroupOptions.RetryPolicy` | *(none; retry is applied per-database via `WithRetry`)* | | `HealthCheckInterval` | `MultiGroupOptions.HealthCheckInterval` | *(none; it is the group's re-evaluation cadence)* | | `FailbackDelay` | `MultiGroupOptions.FailbackDelay` | `ConnectionGroupMember.FailbackDelay` | +| `FailbackHealthCheck` | `MultiGroupOptions.FailbackHealthCheck` | `ConnectionGroupMember.FailbackHealthCheck` | | `Weight` | *(none)* | `ConnectionGroupMember.Weight` | | `SkipInitialHealthCheck` | *(none)* | `ConnectionGroupMember.SkipInitialHealthCheck` | @@ -430,7 +431,7 @@ The probe policy determines how multiple probe results are evaluated to determin #### HealthCheckProbePolicy.AnySuccess -The health check passes if **any** probe succeeds. This provides the most lenient evaluation: +The health check passes if **any** probe succeeds, and fails only if at least one probe failed and none succeeded. This provides the most lenient evaluation. A run in which every probe was `Inconclusive` reports `Inconclusive`, not `Unhealthy`: ```csharp HealthCheck healthCheck = new HealthCheck.Builder @@ -443,7 +444,7 @@ HealthCheck healthCheck = new HealthCheck.Builder #### HealthCheckProbePolicy.AllSuccess (Default) -The health check passes only if **all** probes succeed. This provides the strictest evaluation: +The health check passes only if **no** probe fails and at least one succeeds. This provides the strictest evaluation. A run in which every probe was `Inconclusive` (the probe could not tell) reports `Inconclusive`, not `Healthy`: ```csharp HealthCheck healthCheck = new HealthCheck.Builder @@ -753,7 +754,88 @@ ConnectionGroupMember[] members = [ This guards against **flapping**: a member that is intermittently failing will keep pushing its "last failure" time forward, so it never satisfies the delay and stays out of rotation until it is genuinely stable. -> Implementation note: the failback check is pure tick math on the wall clock — the last-failure time and the cutoff (`UtcNow - FailbackDelay`) are both compared as raw `long` UTC ticks, so `DateTimeKind` never enters into it. (`DateTime.Ticks` and `TimeSpan.Ticks` share the same 100 ns unit.) +> Implementation note: the `FailbackDelay` comparison is pure tick math on the wall clock — the last-failure time and the cutoff (`UtcNow - FailbackDelay`) are both compared as raw `long` UTC ticks, so `DateTimeKind` never enters into it. (`DateTime.Ticks` and `TimeSpan.Ticks` share the same 100 ns unit.) + +### Failback health checks + +`HealthCheck` answers "is this member alive". Some deployments need a second question answered before traffic +moves *to* a member: "has it caught up". In an Active-Active deployment, a region that has just come back can be +reachable but stale, and failing back onto it reads (and builds on) data that is behind. +`ConnectionGroupMember.FailbackHealthCheck` (or the group default, `MultiGroupOptions.FailbackHealthCheck`) is +that second question: + +```csharp +ConnectionGroupMember member = new("us-west.redis.example.com:6379", name: "US West") +{ + Weight = 10, + // CaughtUpProbe stands for a probe of your own; see "Custom Health Check Probes" + FailbackHealthCheck = new HealthCheck.Builder { Probe = new CaughtUpProbe(), ProbeCount = 1 }, +}; +``` + +The rules: + +- **It never removes the active member.** It is consulted only for members that are not active, and only + decides whether they may *become* active. Liveness stays with `HealthCheck`. +- **Only `Healthy` makes a member eligible.** `Inconclusive` (for example, the probe could not reach whatever + it asks) leaves the member ineligible, as does `Unhealthy`; neither marks it unhealthy. +- **A verdict is not kept while it cannot be acted on.** A member that is active, or not connected, is checked + afresh before it is next eligible. +- **If no connected member is eligible, selection falls back to liveness alone.** In a partition between + regions every member can be behind at once, and a group that refused to serve from any of them would turn a + consistency precaution into an outage. When the active member is lost, the best eligible survivor is + preferred, then the best live one. +- **A member chosen that way is provisional.** It is left for an eligible member as soon as one is + available, whatever their weights, and becomes an ordinary active member once it passes its own failback + check. Other clients never select a member that fails its lag check at all; this is the nearest + equivalent that still serves when every member is behind. +- **An explicit `TryFailoverTo(member)` overrides it**, as it does `IsUnhealthy`. + +A probe that asks about the database as a whole, rather than one endpoint, can set +`HealthCheckProbe.Scope` to `HealthCheckProbeScope.Member`, so that it runs once per member per pass rather than +once per endpoint. + +#### Redis Enterprise: lag-aware availability + +For Redis Enterprise Active-Active deployments, `HealthCheck.LagAware(...)` asks the cluster's own +[database availability API](https://redis.io/docs/latest/operate/rs/monitoring/db-availability/) whether a +member is available and caught up. The only thing it needs is the cluster's management credentials: + +```csharp +ConnectionGroupMember member = new("redis-14460.us-west.example.com:14460,password=...", name: "US West") +{ + Weight = 10, + FailbackHealthCheck = HealthCheck.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential("probe@example.com", secrets.Get("us-west-mgmt"))), + }), +}; +``` + +What it does by default, and why: + +- **The REST endpoint is derived** from the member's first configured endpoint: a database named + `redis-{port}.{cluster}` (the form Redis Enterprise assigns) is asked via `https://{cluster}:9443/`. The + cluster name matters, because only the cluster's master answers every route the check uses; other nodes + redirect to its internal address, which is usually unreachable from outside the cluster's network. + Redirects are never followed. Set `RestEndpoint` if the database is reached through a name of your own. +- **The database is found** by matching that endpoint against the cluster's databases, using only filtered + requests (unfiltered database listings include database passwords, even for read-only roles); set + `DatabaseId` to skip this. +- **The lag question is only asked of Active-Active databases** (`LagCheck = LagCheckMode.Auto`). Asked of + any other database, the server reports it unavailable regardless of tolerance. +- **No tolerance is sent**, so the cluster's own `availability_lag_tolerance_ms` applies; set `LagTolerance` + to override it. (Other clients default to 5000 ms; the server's default is 100 ms.) +- **"Could not ask" is `Inconclusive`, not `Unhealthy`.** A management-plane problem (REST API unreachable, + credentials rejected) never fails anything over; as a failback check it simply leaves the member + ineligible. A role with `management: db_viewer` is sufficient. +- **The management certificate is usually self-signed by the cluster**, so it will not validate against the + platform trust store; `options.TrustIssuer(path)` with the cluster's own certificate (or its issuer) works, + as it does for the data plane, but it is a separate setting from the database's TLS trust. + +One limitation to plan around: measured on Redis Enterprise 8.0, when the link between regions breaks, the +lag check keeps reporting "caught up" for roughly a minute before it notices. A failback in that window is not +protected by it; a `FailbackDelay` of at least that long closes the gap. ### Anti-flap tiebreak in selection diff --git a/notes/lag-aware/api-sketch.md b/notes/lag-aware/api-sketch.md new file mode 100644 index 000000000..169039ce0 --- /dev/null +++ b/notes/lag-aware/api-sketch.md @@ -0,0 +1,260 @@ +# Lag-aware availability: API sketch + +Status: **sections 1 to 3 implemented** (2026-10-08): the failback check, member-scoped probes, and the +lag-aware probe (`HealthCheck.LagAware`, `HealthCheckProbe.LagAware`, `LagAwareOptions`, `LagCheckMode`), +with stubbed-transport tests replaying the measured response shapes. The fault-injector scenario test passes +against a live Active-Active pair. Not yet: client-certificate support for the REST call. +Evidence is in [`findings.md`](findings.md) (section 10 especially); sequence in [`plan.md`](plan.md). +*Correction:* `SER007` was retired when geo-redundant failover went GA (#3247), so the shipped +`Availability/` API is not experimental, and nothing here is either. + +Aim: parity with Lettuce, Jedis, redis-py and go-redis where it serves users, and deliberate, +documented deviations where the measurements say the others get it wrong. + +## The shape in one example + +```csharp +var east = new ConnectionGroupMember("redis-14460.c1.example.com:14460,password=...", "east") +{ + // new: consulted only before moving traffic *to* this member (see 1) + FailbackHealthCheck = HealthCheck.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential("probe@example.com", secrets.Get("re-mgmt"))), + }), +}; +``` + +That is the whole opt-in. The REST endpoint, database uid, whether the database is Active-Active, and +the tolerance are all derived or left to the cluster (see 3). Liveness stays with the member's +existing `HealthCheck` (PING by default). + +## 1. Where the lag verdict goes: a failback-only check, not a new result value + +**Proposal:** a second, optional health check per member, consulted only when deciding whether a +member may *become* active. + +```csharp +public sealed class ConnectionGroupMember +{ + public HealthCheck? HealthCheck { get; set; } // existing: liveness, may evict + public HealthCheck? FailbackHealthCheck { get; set; } // new: eligibility, never evicts +} + +public sealed class MultiGroupOptions.Builder +{ + public HealthCheck? FailbackHealthCheck { get; set; } // new: group default, null = none +} +``` + +Rules, applied by `SelectPreferredGroup` / `UpdateState`: + +- **The active member is kept while its liveness check passes**, whatever its failback check says. +- **A member becomes eligible for selection** when its liveness check passes *and* its failback + check (if any) answers `Healthy`. `Inconclusive` from the failback check means **not eligible**: + "could not ask" must not let the group fail back onto a member that may be stale. +- **If no member is eligible, select on liveness alone.** Every member lagging (a partition between + regions, say) must degrade to "serve from somewhere", never to "serve from nowhere". Measured under a + partition (findings section 10): both members' lag checks pass for about a minute, then both fail at + 100 ms. So a region outage that lasts more than a minute leaves *no* member passing the lag check; + a strict rule (go-redis's) would allow failover in the first minute and refuse it afterwards. In + short: the failback check gates moving *away from a live active member*; it never blocks replacing a + dead one. +- **The first minute after a link breaks is blind** (measured): the lag check reports no lag for + ~60 s after sync stops. So a failback decision in that window is not protected by the lag check; + `FailbackDelay` covers some of it, and the docs must say so. +- The failback check runs on the same `HealthCheckInterval` poll as liveness, but only for members + that are not active (the active member's answer is never used). + +**Why not the new `HealthCheckResult` value** (accepted in principle, but this is better, I think): + +- One check would have to answer two different questions: "is it alive" (PING) and "is it caught up" + (REST). A new value means one probe that does both, so every lag-aware user also takes over + liveness. +- Every consumer of `HealthCheckResult` would have to learn the new value: `CollateAsync`, + `UpdateState`, all three `HealthCheckProbePolicy` implementations, and any user-written probe or + policy. Measured: they currently treat everything except `Unhealthy` as healthy, so a value they do + not know would silently mean "fine". +- It matches go-redis's `FailbackOnly()`, the only client that addresses this at all. + +**Parity note:** Lettuce, Jedis and redis-py put the lag check in the single health-check slot, so a +stale reading evicts the active member. Measured: only the member *behind* flips, which is never the +one being written to, so for a single-writer deployment that eviction does not happen, and for a +multi-writer one it would be wrong. This is a deliberate deviation. + +**As built.** One refinement, decided 2026-10-08 by "do what the other clients do": a member chosen by +the liveness fallback is **provisional**. It is left for an eligible member as soon as one is available +(whatever the weights), and its own failback check keeps running so that it becomes an ordinary active +member once it passes. Rationale: Lettuce, Jedis and redis-py treat a lagging database as unhealthy, and +go-redis never selects a lagging candidate, so none of them can end up on an unverified member while a +verified one exists; this is the nearest equivalent that still serves when every member is behind. +Tested in `FailbackHealthCheckTests`; the fixture holds all members down until every connection is up, so +that a start-up race does not pick a provisional member before the test means it to. + +Implementing it also surfaced an existing bug: `HealthCheckProbePolicy.AllSuccess` (the default) reported +`Healthy` for a run of nothing but `Inconclusive` probes, contrary to its own comment. Harmless for +liveness (both count as connected) but fatal for a failback check, where "could not ask" would have read as +"caught up". Fixed: such a run is now `Inconclusive`. The mirror-image bug in `AnySuccess` (all +`Inconclusive` reads as `Unhealthy`, which *does* affect liveness) is left alone for a separate decision. + +## 2. Running once per member, not per endpoint + +`HealthCheck.CheckHealthAsync(IConnectionMultiplexer)` calls the probe once per server. A REST probe +answers per database, so an OSS-cluster member with six endpoints would make six identical calls. + +```csharp +public enum HealthCheckProbeScope { Endpoint = 0, Member = 1 } + +public abstract class HealthCheckProbe +{ + public virtual HealthCheckProbeScope Scope => HealthCheckProbeScope.Endpoint; // new +} +``` + +A `Member`-scoped probe is called once per pass, with `HealthCheckContext.Server` set to a connected +server of that member (needed for the configured endpoint and its `RawConfig`). Adding a virtual +member is binary-safe for existing subclasses. + +## 3. The probe + +```csharp +public sealed partial class HealthCheck // existing type; new static factories +{ + public static HealthCheck LagAware(LagAwareOptions options); // extend_check=lag (if Active-Active) + public static HealthCheck DatabaseAvailability(LagAwareOptions options); // plain availability only +} + +public sealed class LagAwareOptions +{ + // required: the one thing that cannot be derived + public Func> Credentials { get; set; } + + // optional; null = derive or defer + public Uri? RestEndpoint { get; set; } // null: https://:9443 + public int? DatabaseId { get; set; } // null: discover by host and port + public LagCheckMode LagCheck { get; set; } // default Auto + public TimeSpan? LagTolerance { get; set; } // null: send none, cluster setting applies + public RemoteCertificateValidationCallback? CertificateValidation { get; set; } + public void TrustIssuer(string path); // same helper as ConfigurationOptions + public void TrustIssuer(X509Certificate2 issuer); +} + +public enum LagCheckMode { Auto = 0, Enabled = 1, Disabled = 2 } +``` + +The `HealthCheck` returned is an ordinary `HealthCheck` (so `ProbeCount`, `ProbeTimeout` and +`ProbeInterval` apply, via `new HealthCheck.Builder(HealthCheck.LagAware(...)) { ... }`), whose +probe is `Member`-scoped. + +### What it does, per pass + +1. **Endpoint:** `RestEndpoint`, else derived from the member's first configured endpoint: the cluster + FQDN when the host is `redis-.`, else the host itself. *As built:* the cluster + FQDN, not the database host, because non-master nodes redirect management routes to the master's + internal address (findings section 10); redirects are never followed. Configured endpoint, not + discovered nodes: an OSS-cluster database advertises IP addresses. +2. **uid** (once, cached; cleared on 404, as Lettuce does): `DatabaseId`, else + `GET /v1/bdbs?fields=uid,endpoints`, matched on `dns_name` and port. +3. **Active-Active** (once, cached), for `LagCheck = Auto`: `GET /v1/bdbs/{uid}?fields=uid,crdt`. +4. **Check:** `GET /v1/bdbs/{uid}/availability`, adding `extend_check=lag` when the database is + Active-Active (`Auto`) or `Enabled`, and `availability_lag_tolerance_ms` only if `LagTolerance` is set. + +**Never an unfiltered `/v1/bdbs` or `/v1/crdbs`.** Measured: those return database passwords even to +`db_viewer` when `mask_bdb_credentials` is off. + +### Result mapping + +| response | result | note | +| --- | --- | --- | +| 200 | `Healthy` | body is empty | +| 503 (any `error_code`) | `Unhealthy` | lag and "unavailable" share `bdb_unavailable` | +| 404 | `Unhealthy` | and clear the cached uid | +| 401, 403, other status | `Inconclusive` | management-plane problem; logged | +| connect / TLS failure, timeout | `Inconclusive` | as above | + +The last two rows are the deviation from every other client (all map REST failure to unhealthy). As a +failback check, `Inconclusive` already means "not eligible" (1), so the deviation only changes the +outcome where it should: a management-plane outage can no longer evict anything. + +### Defaults compared + +| | Lettuce | Jedis | redis-py | go-redis (branch) | **this** | +| --- | --- | --- | --- | --- | --- | +| activation | explicit | explicit | explicit | explicit | explicit | +| REST endpoint | supplied | supplied | supplied | derived from db host | **derived**, overridable | +| uid | discovered | discovered | discovered (unfiltered) | discovered | **discovered (filtered)**, overridable | +| lag check | on | on | on | on | **only if Active-Active** (`Auto`) | +| tolerance | 5000 ms | 5000 ms | 5000 ms | 5000 ms | **unset: cluster's own** (100 ms measured) | +| REST failure | unhealthy | unhealthy | unhealthy | unhealthy | **inconclusive** | +| evicts active member | yes | yes | yes | no (failback only) | **no (failback only)** | +| credentials | rotating supplier | supplier | static | static | **rotating callback** | +| TLS | own truststore | own truststore | own CA / verify | own `tls.Config` | **same callback type as Redis**, separate setting | + +## 4. Activation + +Explicit, as in every other client: setting `FailbackHealthCheck`. Management credentials are a +separate secret from the Redis password, so no options provider can supply them, and nothing else +needs configuring. What a provider *can* do, later and separately: `RedisEnterpriseOptionsProvider` +could carry a group-wide default once credentials have a home in configuration. Not proposed now. + +## 5. Platform + +| TFM | | +| --- | --- | +| `net8.0`, `net10.0` | everything | +| `net472`, `netstandard2.0` | everything; JSON (steps 2 and 3, and `error_code` for logging) via a `System.Text.Json` package reference on these targets only - in-box on `net8.0`+. *Decided 2026-10-07:* NRedisStack, which builds on SE.Redis, already depends on it | +| `net461` | works with ambient trust; setting `CertificateValidation`/`TrustIssuer` throws `PlatformNotSupportedException` at creation, on the understanding that `ServerCertificateCustomValidationCallback` needs .NET Framework 4.7.1 (*unverified*; the build will say) | + +Test seam: an internal `Func` on `LagAwareOptions`, reached via +`InternalsVisibleTo`, so tests replay captured responses without a listener. Kept internal so no +`System.Net.Http` type enters the public API; Lettuce's public `HttpClient` constructor shows there is +demand, and it can be made public later. + +## Open questions, compared with the other clients + +Read from source 2026-10-07: Lettuce `114a3ef`, Jedis `bb5f01d`, redis-py `c3bb0e6`, go-redis +`feature/multidb-integration` `9377b5b`. (Correction to the earlier survey: redis-py's sync client +imports the same `LagAwareHealthCheck`, so it is not asyncio-only.) + +1. ~~**JSON on older targets**~~ *Decided:* `System.Text.Json`, see 5. + +2. **Names.** + + | | the check | general abstraction | returning to the preferred database | failback-only check | + | --- | --- | --- | --- | --- | + | Lettuce | `LagAwareStrategy` (`databaseAvailability` / `lagAware` / `lagAwareWithTolerance`) | `HealthCheckStrategy` | `failbackSupported`, `failbackCheckInterval`, `gracePeriod` | none | + | Jedis | `LagAwareStrategy`, same factories | `HealthCheckStrategy` | same three as Lettuce | none | + | redis-py | `LagAwareHealthCheck` | `HealthCheck` | `auto_fallback_interval`, `grace_period` | none | + | go-redis | `LagAwareHealthCheck`, `NewLagAwareHealthCheck(WithLagAware...)` | `MultiDBHealthCheck` | `AutoFallbackInterval`, `GracePeriod` | `FailbackOnly() bool`, a marker on the check | + + *Recommendation:* `LagAware` is universal, keep it, with `DatabaseAvailability` alongside as + Lettuce/Jedis have. Use "failback", not "fallback": it is our existing word (`FailbackDelay`) and + the Java clients'. `FailbackHealthCheck` as a separate slot rather than go-redis's marker, because + our member has one `HealthCheck` rather than a list, and a slot keeps liveness independent. + +3. **Credentials.** + + | | type | rotates | client certificate to REST | + | --- | --- | --- | --- | + | Lettuce | `Supplier`, read per request | yes | via `SslOptions` keystore (*inferred*) | + | Jedis | `Supplier`, read per call | yes | via `SslOptions` keystore (*inferred*) | + | redis-py | `auth_basic: Tuple[str, str]` | no | `client_cert_file` / `client_key_file` / `client_key_password` | + | go-redis | `WithLagAwareBasicAuth(user, password)` | no | `WithLagAwareClientCert(PEM)` / `...FromFiles` | + + *Recommendation:* keep the callback returning `NetworkCredential` (rotation parity with the Java + clients; in-box everywhere), and add client-certificate support as its own option, mirroring + `ConfigurationOptions`' existing `CertificateSelection` callback and PEM/PFX helpers, as every client + supports it. No dedicated credentials type needed. + +4. **When every member fails.** + + | | active member's own lag check fails | everything unhealthy | "all lagging" special-cased | + | --- | --- | --- | --- | + | Lettuce | evicted | `RedisNoHealthyDatabaseException`, `AllDatabasesUnhealthyEvent`, retries forever | no | + | Jedis | evicted | `JedisTemporarilyNotAvailableException`, then `...PermanentlyNotAvailableException` | no | + | redis-py | evicted | `NoValidDatabaseException` inside `TemporaryUnavailableException` | no | + | go-redis | **kept** (PING judges it) | stays on a live active; otherwise `ErrTemporarilyNotAvailable`, then `ErrPermanentlyNotAvailable` | no; lagging candidates stay unselectable | + + *Recommendation:* no client falls back to liveness alone, so the proposal in 1 has no precedent. + Keep it anyway. *Measured 2026-10-08* by partitioning the clusters: after ~60 s every member fails + the lag check, so go-redis's rule would refuse a failover during any region outage longer than a + minute (its behaviour there is *inferred* from its code, not run). diff --git a/notes/lag-aware/findings.md b/notes/lag-aware/findings.md new file mode 100644 index 000000000..26ccdb84a --- /dev/null +++ b/notes/lag-aware/findings.md @@ -0,0 +1,759 @@ +# Lag-aware availability checks + +Investigation, September 2026. Does Redis Enterprise's lag-aware database-availability API belong in +our health checks for geo-redundant failover, and if so, where? The sequence that follows from this +is in [`plan.md`](plan.md). + +Short answer: **yes, it is the same feature every other Redis client has already shipped, and our +abstractions already fit it — but it needs HTTP and JSON, which `src/` has neither of, and it does +not need to live in `StackExchange.Redis` at all.** + +## 1. What the feature is + +Redis Enterprise (Redis Software) exposes a +[database availability API](https://redis.io/docs/latest/operate/rs/monitoring/db-availability/) for +load balancers and monitoring: + +```text +GET /v1/bdbs/{uid}/availability # whole database +GET /v1/local/bdbs/{uid}/endpoint/availability # this node's local endpoint, no redirect +``` + +`200 OK` means available. An endpoint counts as available only if the database's primary shards are +reachable *and* the endpoint's listener port is bound. With the OSS Cluster API enabled, the +database form verifies **all** endpoints; otherwise at least one. + +The **lag-aware** extension is what makes this interesting for failover, as opposed to a liveness +probe: + +```text +GET /v1/bdbs/{uid}/availability?extend_check=lag +GET /v1/bdbs/{uid}/availability?extend_check=lag&availability_lag_tolerance_ms=100 +``` + +`extend_check=lag` additionally asks whether a replica is *sufficiently synchronised with the +primary* to be a safe failover/failback target. The threshold is cluster-wide, default **100 ms**, +changed with: + +```text +PUT /v1/cluster { "availability_lag_tolerance_ms": 100 } +``` + +and overridable per request via the query parameter. The REST docs say "Recommended value: 100 +milliseconds". + +The purpose is explicitly disaster recovery: *"reduce the risk of data inconsistencies during +disaster recovery by … ensuring failover-failback flows only occur when databases are accessible and +sufficiently synchronized."* This is the check that stops you failing **back** onto a region that is +reachable but stale. + +> **Corrected by measurement (§10).** On RS 8.0.22 the lag is Active-Active (CRDB) sync lag, not +> intra-cluster replica lag. On any database that is not an Active-Active member, `extend_check=lag` +> answers `503 bdb_unavailable` unconditionally - with or without `replication`, and whatever +> `availability_lag_tolerance_ms` says - while the plain check answers 200. The "replica" wording +> above is the documentation's, and it is misleading. + +### Response detail + +| status | meaning | +| --- | --- | +| 200 | available | +| 400 | invalid schema | +| 404 | database not found | +| 503 | unavailable, or no quorum | + +Non-200 returns a JSON body with `error_code` and `description`: + +| form | `error_code` values | +| --- | --- | +| database | `no_quorum`, `db_not_found`, `bdb_unavailable` — suffixed `_shard_unreachable` and/or `_port_unbound`, e.g. `bdb_unavailable_shard_unreachable_port_unbound` | +| endpoint | `no_quorum`, `db_not_found`, `bdb_endpoint_unavailable` | + +Authentication is HTTP Basic against the **cluster REST API** (default port 9443), requiring the +`view_bdb_info` permission — roles `admin`, `cluster_member`, `cluster_viewer`, `db_member`, +`db_viewer`, `user_manager`. Note these are *cluster management* credentials, unrelated to the Redis +credentials in `ConfigurationOptions`. + +Documented known issue: **RS155734** — endpoint availability metrics are miscalculated. + +## 2. The reference implementation + +[`LagAwareStrategy`](https://github.com/redis/lettuce/blob/main/src/main/java/io/lettuce/core/failover/health/LagAwareStrategy.java) +in Lettuce implements `HealthCheckStrategy`, whose surface is `getInterval()`, `getTimeout()`, +`getNumProbes()`, `getPolicy()`, `getDelayInBetweenProbes()`, `doHealthCheck(RedisURI)` and +`close()`. + +`doHealthCheck` is short: + +1. If no cached bdb id: `GET /v1/bdbs?fields=uid,endpoints`, find the first bdb whose endpoints match + the connection's host, cache its `uid`. No match → log and throw. +2. `GET /v1/bdbs/{uid}/availability`, with `extend_check=lag` and + `availability_lag_tolerance_ms` when extended checking is on. +3. 200 → `HEALTHY`, anything else → `UNHEALTHY`. +4. On REST failure → **invalidate the cached bdb id** and return `UNHEALTHY`. + +Config (`LagAwareStrategy.Config`): `restEndpoint` (URI), `credentialsSupplier` +(`Supplier` — so credentials can rotate), `sslOptions`, `availabilityLagTolerance`, +`extendedCheckEnabled`. Convenience factories `databaseAvailability(...)` (plain), +`lagAware(...)`, `lagAwareWithTolerance(...)`. + +Two defaults worth noting: `EXTENDED_CHECK_DEFAULT = true` (lag-aware is on by default), and +`AVAILABILITY_LAG_TOLERANCE_DEFAULT = Duration.ofMillis(5000)`. + +## 3. Every client has this, and the defaults disagree + +This is not a Java curiosity — it is the client-side geographic failover feature family that Redis +has been rolling out across clients: + +| client | type | lag tolerance default | +| --- | --- | --- | +| Lettuce | `LagAwareStrategy` | **5000 ms** | +| Jedis | `LagAwareStrategy` | **5000 ms** | +| redis-py | `LagAwareHealthCheck` | **5000 ms** (*corrected*: its docstring says 100) | +| Redis Enterprise cluster | `availability_lag_tolerance_ms` | **100 ms** | +| REST API docs | "Recommended value" | **100 ms** | + +> **Corrected 2026-10-06, read from source rather than docs** (heads as of that date; `gh` search of +> issues/PRs was blocked by the org's SSO, so discussion that did not reach code is not covered): +> +> - Lettuce `LagAwareStrategy.java:163-165`: `EXTENDED_CHECK_DEFAULT = true`, tolerance 5000 ms. +> Unchanged since it was introduced (#3576, 2026-02-06); #3914 (2026-09-15) promoted it to GA without +> touching either value. +> - Jedis `mcf/LagAwareStrategy.java:96-97`: the same `true` / 5000. +> - redis-py `asyncio/multidb/healthcheck.py:73`: `DEFAULT_LAG_AWARE_TOLERANCE = 5000`, while line 502's +> docstring says "(default: 100)" - both from the same commit, `f3806fad`, 2025-10-07. It always sends +> `extend_check=lag` with the tolerance; there is no plain mode. +> - None checks `crdt`, so given §10, *inferred*: all three report every non-Active-Active database as +> permanently unhealthy under default settings. +> - All three map any REST failure to unhealthy; none has a third state (§4). +> - TLS is truststore/CA based in all three; none supports pinning, so with the cluster's self-signed 9443 +> certificate a user must import that leaf as a trust anchor. +> +> So the server and docs say 100 ms and every client says 5000 ms. Our plan sidesteps it: send no +> tolerance unless configured, and let the cluster's own setting apply. + +**Lettuce is fifty times more permissive than everyone else, including the server's own default and +its own documentation's recommendation.** That is either a deliberate call about real-world WAN +replication lag or an oversight; either way we should not copy a number without deciding it, and it +is worth asking Redis which is intended. 100 ms across a geo-replicated WAN link looks optimistic; +5 s looks like it would permit a failback that loses seconds of writes. + +redis-py's other defaults are useful reference points: `rest_api_port` 9443, `verify_tls` True, with +`auth_basic`, `ca_file`, `client_cert_file` and `client_key_file` — i.e. mTLS is supported, not just +Basic. + +## 4. It fits what we already have — almost exactly + +`src/StackExchange.Redis/Availability/` already implements this shape, gated behind +`[Experimental(Experiments.GeoRedundantFailover)]` (`SER007`). The correspondence with redis-py's +model is close to one-to-one: + +| concept | StackExchange.Redis | redis-py | Lettuce | +| --- | --- | --- | --- | +| probe abstraction | `HealthCheckProbe.CheckHealthAsync(HealthCheckContext)` | `AbstractHealthCheck.check_health()` | `HealthCheckStrategy.doHealthCheck(RedisURI)` | +| default probe | `HealthCheckProbe.Ping` | `PingHealthCheck` | ping strategy | +| aggregation policy | `HealthCheckProbePolicy.AllSuccess` / `AnySuccess` / `MajoritySuccess` | `HEALTHY_ALL` / `HEALTHY_ANY` / `HEALTHY_MAJORITY` | `ProbingPolicy` | +| probe count / timeout / interval | `HealthCheck.ProbeCount` / `.ProbeTimeout` / `.ProbeInterval` | `MultiDbConfig` | `getNumProbes()` / `getTimeout()` / `getInterval()` | +| failure latch | `CircuitBreaker` | circuit breaker OPEN | — | +| failover unit | `ConnectionGroupMember` | database in multi-db config | — | +| failback damping | `MultiGroupOptions.FailbackDelay` | weighted selection | — | + +Two details that make a lag-aware probe land cleanly: + +- **`HealthCheckProbe` is externally subclassable today.** It is a `public abstract partial class` + with one `public abstract` method and *no* internal or `private protected` abstract members — + checked. Anyone can implement a probe outside this assembly. The only in-assembly conveniences are + the memoised `protected internal` result tasks, which an external probe can trivially do without. +- **Health checks can already be configured per member.** `ConnectionGroupMember` carries nullable + per-member overrides of the group-wide `MultiGroupOptions`, health check included + (`ResolveHealthCheck(options) => HealthCheck ?? options.HealthCheck`). That matters, because each + geo member is a *different* Redis Enterprise cluster with its own REST endpoint, its own + credentials and its own bdb uid — so per-member probe configuration is exactly the shape needed, + and it already exists. + +### Where we are better placed than Lettuce + +`HealthCheckResult` has three states — `Healthy`, `Unhealthy`, **`Inconclusive`** — where Lettuce's +`HealthStatus` has two. + +That difference is more than cosmetic here. Lettuce maps *"the REST call failed"* to `UNHEALTHY`, +which means a Redis Enterprise **management-plane** outage (REST API down, credentials expired, +9443 firewalled) is indistinguishable from the **data plane** being unavailable — and can therefore +trigger a failover of a database that is serving traffic perfectly well. With `Inconclusive` we can +say "I could not determine this" and let the policy decide, which is the honest answer and the safer +one. `KeyWriteHealthCheckProbe` already uses `Inconclusive` this way for replicas. + +This is the single most valuable thing we could do differently, and it should be a deliberate, +documented deviation rather than an accident. + +## 5. The awkward part: HTTP and JSON + +`src/` contains **no HTTP client and no JSON parser** — checked across `StackExchange.Redis` and +`RESPite`; the only matches are in `obj/` transitive restore graphs. A lag-aware probe needs both: + +- **HTTP.** `System.Net.Http` is in-box on `net8.0`/`net10.0` and available on `net472` / + `netstandard2.0`. Manageable, but it is a new dependency class for this library, and it drags in + TLS configuration, proxy behaviour, timeouts and connection pooling as things we would own. +- **JSON.** Needed only for `GET /v1/bdbs?fields=uid,endpoints` discovery and for parsing + `error_code` out of failure bodies. `System.Text.Json` would be a new package reference on + `net472`/`netstandard2.0`. + +**JSON is avoidable on the hot path.** The availability call itself is a *status code* check — 200 +versus not-200 — with no body parsing required for the healthy case. If the bdb uid is supplied by +configuration rather than discovered, the required path needs no JSON at all, and `error_code` can be +surfaced as an opaque string for diagnostics. Discovery-by-host (Lettuce's step 1) is the only part +that genuinely needs a parser, and it is optional convenience: anyone configuring a REST endpoint, +credentials and a tolerance already knows their database. + +That suggests an explicit-uid-first design, with discovery as a later, optional extra. + +## 6. Options + +**A — in `StackExchange.Redis`.** Ships a `HealthCheckProbe.LagAware(...)` alongside `Ping` and +`StringSet`. Most discoverable; matches what Lettuce/Jedis/redis-py do (all in-box). Costs the core +package an `HttpClient` dependency, plus TLS/proxy/credential surface, for a feature that only +applies to one commercial deployment target. + +**B — a separate package.** `StackExchange.Redis.Enterprise` (or similar) subclassing the *already +public* `HealthCheckProbe`. **This requires no change to `StackExchange.Redis` whatsoever** — the +extension point exists and is sufficient. Keeps HTTP, JSON, TLS and cluster credentials out of the +core package and lets the Enterprise-specific bits version on their own cadence. + +**C — do nothing, and document.** The extension point is public; users targeting Redis Enterprise +can write a ~50-line probe. Cheapest, and the least good discovery story for the people who most +need it. + +B looks strongest on the evidence: the feature is deployment-specific, the dependency is real, and — +unlike the OpenTelemetry case, where a second assembly would have needed internals it could not have +— the hook here is *already public and already sufficient*. Worth confirming that nothing in +`HealthCheckContext` is missing for a real implementation before committing to it. + +## 7. Testing: the toy server can spoof this, and already has the HTTP half + +`toys/KestrelRedisServer` **already listens on HTTP**. From `Program.cs`, one Kestrel host serves +both planes over the same `RedisServer` singleton: + +```csharp +options.ListenLocalhost(5000); // "HTTP 5000 (test/debug API only)" +options.ListenLocalhost(ip.Port, b => b.UseConnectionHandler()); // RESP 6379 +``` + +with a single catch-all route today (`app.Run(ctx => ctx.Response.WriteAsync(server.GetStats()))`). +Adding `/v1/bdbs/{uid}/availability` beside that is a handful of lines, and — because the route +closes over the *same* `RedisServer` instance the client is talking RESP to — the fake management +plane can be made to lie **coherently with** the data plane rather than independently of it. + +Spoofing infrastructure for test purposes is also already the established intent of this toy; the +file carries a commented-out block headed *"demonstrate cluster spoofing"* that flips +`ServerType` to `Cluster`, adds an empty node and migrates a slot. A fake Redis Enterprise +management plane is the same idea applied to a different API. + +What that buys is the ability to script the scenarios that decide whether the failover logic is +correct, none of which can be produced on demand against a real cluster: + +- healthy (200) +- `503 bdb_unavailable_shard_unreachable`, `503 no_quorum`, `404 db_not_found` +- **200 for the plain check, 503 for `extend_check=lag`** — reachable but stale. *This asymmetry is + the entire point of the feature.* (See the fault injector below: this one **can** also be produced + against a real cluster, contrary to what one would assume.) +- slow responses, to exercise `HealthCheck.ProbeTimeout` +- 401/403, to exercise credential rotation and — importantly — to check we return `Inconclusive` + rather than `Unhealthy` when the management plane is the thing that is broken (§4) + +### Two levels, and a design consequence + +The existing test harness points at the right answer for the cheap level — but the two planes are +**not** symmetric, and this is the thing to get right. + +`InProcessTestServer` derives from `MemoryCacheRedisServer` and sets `Tunnel = new InProcTunnel(this)`. +It **does not run as a network server at all**: there is no listener, no port, no socket. The client +reaches it because `ConfigurationOptions.Tunnel` replaces the transport wholesale, so "connecting" is +a method call. That is what makes those tests fast and deterministic. + +This is not a hack bolted on for tests — it is a first-class, public, documented extension point. +`StackExchange.Redis.Configuration.Tunnel` is a `public abstract class` whose +`ConnectTransportAsync(...)` returns a `RESPite.Transports.DuplexTransport`, i.e. an implementation +**hijacks the transport layer completely**; the shipped `Tunnel.HttpProxy(...)` does the same thing +for CONNECT proxying. `InProcTunnel` is simply another implementation of that same public seam. + +There is no such seam on the HTTP side. `HttpClient` resolves a URL and opens a socket, so **you +cannot point a lag-aware probe at the in-process server — there is nothing there to point at.** + +So the model to copy is the one the library already has: hijack the HTTP transport completely, the +same way `Tunnel` hijacks the RESP one. In .NET that means an injectable `HttpMessageHandler` (or our +own abstraction over it), which is the exact analogue — `HttpMessageHandler` *is* the HTTP transport, +and substituting it answers requests in-process with no listener, no port and no TLS. + +Hence the sharper version of the design consequence below: a configurable *base URL* is not enough. +If the probe's only seam is a `Uri`, every test needs a real listener on a real port, and the cheap +in-process level is simply unavailable. Lettuce evidently hit the same wall — its second constructor, +`LagAwareStrategy(Config, HttpClient)`, exists so the transport can be substituted. + +`toys/KestrelRedisServer` then covers the other level: a real, out-of-process, full-fidelity fake with +actual sockets on both planes, for manual exercise and end-to-end runs. Note that this is also the +only level where the *whole* thing is exercised, because it is the only place both planes are real. + +The combination is what makes this worth doing: **in-process RESP fakes for several members, plus a +stubbed management plane per member, is a complete geo-redundant failover test with no external +infrastructure at all** — and geo-redundant failover is otherwise close to untestable, which is +presumably why it is still behind `SER007`. + +### Real servers are available, and they answer different questions + +mgravell notes we have access to real servers that will support this. That does not replace the +fakes; it does something the fakes cannot, and it should come **first**. + +Everything in §1 of these notes is read off *documentation*, and documentation for a recent feature +is exactly where reality and prose diverge. A short session against a real cluster settles a list of +things currently taken on trust: + +- does the deployed version support `extend_check=lag` at all, and does the + `availability_lag_tolerance_ms` query parameter genuinely override the cluster default? +- what does a failure body actually look like — are the composed codes + (`bdb_unavailable_shard_unreachable_port_unbound`) real, and is `error_code` reliably present? +- is a `Host: cnm.cluster.fqdn` header actually required, as the docs' header table implies? + (`ClusterRestClient` sets no such header and works, so: apparently not — but it only issues + `GET /v1/bdbs`, so worth re-checking on the availability route.) +- **what lag do real geo-replicated links actually show?** This is the empirical way to settle + 100 ms versus Lettuce's 5000 ms (§3), and it is a much better argument than reading either + default off a page. + +The two then compose rather than compete, and the synthesis is the useful bit: **use the real +cluster to capture responses, and the fakes to replay them.** A recorded set of genuine 200/503/404 +bodies baked into the stub means the deterministic tests stop guessing at the wire format. + +### The fault injector — and we already have one + +I had assumed a real cluster could not be made to show "reachable but stale" on demand. **That is +wrong twice over**: Redis ships a fault injection service for exactly this, *and we already have a +test tier built on it.* + +**`tests/StackExchange.Redis.FaultInjector.Tests`** exists on `marc/maint-optin-server`, open as +**[#3191](https://github.com/StackExchange/StackExchange.Redis/pull/3191)** (107 files, +13,160, +unmerged at the time of writing). Per its README it drives *"a real Redis Enterprise deployment +through the fault injector, and watch[es] how SE.Redis reacts … the tier that can observe what no +in-process fake can: real DNS, real TLS identity, real timing."* Gating is +`SER_FI_CONFIG_DIR` plus an explicit `E2E_SCENARIO_TESTS=true` opt-in, with `FAULT_INJECTION_API_URL` +defaulting to `http://127.0.0.1:20324`. + +Two pieces of it matter directly here. + +**`FaultInjector/FaultInjectorClient.cs`** — `POST /action` returning an id, `GET /action/{id}` to +poll, with its own note that this is *"the same shape go-redis, redis-py and node-redis wrap … +they integrated independently and converged, so the contract is stable"*. It is richer than Lettuce's: +alongside `StartActionAsync` / `RunActionAsync` / `WaitForActionAsync` there is +`GetValidTriggersAsync(scenario, effect, clusterIndex)`, so **we can ask the injector what +lag-inducing effects the deployment actually supports rather than guessing at an action name.** +Lettuce's own Active-Active failover test uses `network_latency` with `bdb_id`, `delay_ms` and +`duration`, which is the obvious candidate to look for. + +**`Environment/ClusterRestClient.cs`** — *already a Redis Enterprise REST client*, described as "the +cluster's own REST API on port 9443, for the few facts the fault injector does not expose". It +already does `GET /v1/bdbs?fields=uid,name`, with HTTP Basic auth and `System.Text.Json`. Adding +`GET /v1/bdbs/{uid}/availability?extend_check=lag` beside it is a method, not a project. + +That client also **answers several questions in §9 empirically**, which is worth more than the +documentation they were drawn from: + +- **TLS on 9443 is self-signed per environment.** The client pins to a CA from the config directory + via `ServerCertificateCustomValidationCallback` with `X509ChainTrustMode.CustomRootTrust`, with the + comment *"this channel carries credentials"*. So a lag-aware probe **does** need CA/cert + configuration in the redis-py mould; relying on the ambient trust store will not do. + *Corrected (§10):* the 9443 certificate is self-signed by the cluster itself (issuer = subject = + cluster FQDN, minted at bootstrap), and the CA in the config directory signs only the RESP-plane + proxy certificates - so that chain build could never succeed. `ClusterRestClient` now pins the leaf + on first use instead. The conclusion stands and sharpens: a probe needs a pinned certificate or + fingerprint option, not merely a custom CA. +- **Auth is HTTP Basic**, confirmed against a real cluster rather than inferred from Lettuce. +- **`/v1/bdbs?fields=…` returns an array of objects carrying the requested fields**, so Lettuce's + `fields=uid,endpoints` discovery is straightforward. That weakens the §5 argument a little: skipping + discovery still keeps JSON out of **`src/`**, but "discovery is hard" is not the reason to skip it. + +The fakes are demoted, not removed: they still cover machines with no Enterprise cluster, they are +far faster (injector actions budget minutes, plus a stabilisation wait), and they reach what the +injector does not obviously reach — specific `error_code` bodies, 401/403 credential expiry, and +slow-but-successful responses for probe-timeout behaviour. + +**Sequencing consequence:** an end-to-end lag-aware test is downstream of #3191. That is a real +dependency to plan around, not a detail. + +And #3191's own write-up is the strongest possible support for validating §1 before writing code: +*"the specifications are prose, the payloads were never published, and nearly every assumption I +started with was wrong in some way that mattered."* Same vendor, same class of API, same trap. + +Gating is a solved problem here: `tests/.../Helpers/Skip.cs` already has `IfNoServer(host, port)`, +`IfNoCluster()`, `IfNoFailoverPair()` and `UnlessLongRunning()`, and `TestConfig` carries per-role +host/port settings, so an Enterprise-backed test suite follows an established pattern and skips as +inconclusive wherever the cluster is absent — which is everywhere except the machines that have one. + +The design consequence: **whatever we build must be able to hijack the HTTP transport completely, +from day one** — an injectable `HttpMessageHandler`-shaped seam, not merely a configurable base URL. +That is a requirement on the API shape, it falls out of testability rather than taste, and it is the +same decision the library already made for RESP with `Tunnel`. It also argues mildly against option +C — "document the extension point and let users write it" leaves us with no test coverage of a +failover path we ship. + +## 8. If `Tunnel` gains an API for this + +mgravell has offered to add one. It is a good fit — but the *shape* decides whether it helps or +quietly undoes §6. + +### Two functional arguments for `Tunnel` specifically + +1. **The proxy case is already right.** If Redis traffic goes through a CONNECT proxy + (`Tunnel.HttpProxy(...)`), the Redis Enterprise REST API is almost certainly behind the same + corporate proxy. Hanging the management-plane transport off the same object means that case works + by construction instead of needing a second, parallel proxy setting that users must remember to + keep in sync. +2. **One assignment configures both planes in tests.** `InProcessTestServer` already sets + `Tunnel = new InProcTunnel(this)`. If the HTTP seam rides on the same object, the harness gets + management-plane spoofing everywhere it already gets data-plane spoofing, with no new wiring. + +### The shape that would be a mistake + +```csharp +// don't +public virtual HttpMessageHandler? CreateHttpMessageHandler(Uri restEndpoint); +``` + +That puts `System.Net.Http` types into **StackExchange.Redis's public API**, giving the core package +a hard dependency for the sake of one commercial deployment target — exactly what option B exists to +avoid. It would make the core-vs-package question moot by deciding it the expensive way. + +### The shape that works + +Stay transport-neutral. `Tunnel` yields a `Stream` (or `DuplexTransport`) for an endpoint, and the +Enterprise-side package composes that into an HTTP stack itself: + +```csharp +// core: no System.Net.Http anywhere +public virtual ValueTask ConnectManagementStreamAsync( + EndPoint endpoint, CancellationToken cancellationToken) => default; + +// Enterprise package: hand it to SocketsHttpHandler +new SocketsHttpHandler +{ + ConnectCallback = async (ctx, ct) => + await tunnel.ConnectManagementStreamAsync(Resolve(ctx.DnsEndPoint), ct) + ?? await DefaultConnectAsync(ctx, ct), +}; +``` + +`SocketsHttpHandler.ConnectCallback` is `Func>` — a `Stream` is precisely the currency needed, so a stream-shaped `Tunnel` member +composes into a complete HTTP hijack with no HTTP types in core at all. `Tunnel` already speaks this +language: `BeforeAuthenticateAsync` returns `ValueTask` and `ConnectTransportAsync` returns +`ValueTask` (itself public, gated `SER009`). + +Adding a **virtual** member to a public abstract class is source- and binary-safe for existing +subclasses, so this is additive in the sense AGENTS.md requires. + +### Caveats worth stating + +- **`ConnectCallback` is net5.0+.** There is no equivalent on `HttpClientHandler`/`WinHttpHandler`, + so a down-level implementation would need direct handler injection instead. An Enterprise package + targeting `net8.0`+ sidesteps this entirely, and is defensible for a new deployment-specific + feature. +- **Signature mismatch.** Existing `Tunnel` members are RESP-connection-shaped (`EndPoint` plus + `ConnectionType`). HTTP wants scheme, host, port and TLS — i.e. a `Uri`. Either shape is workable; + neither is free of a little awkwardness. +- **It widens `Tunnel`'s remit** from "the Redis connection" to "transports this library initiates", + which the existing subclasses then have to have an opinion about. `HttpProxyTunnel` almost + certainly *should* apply to the management plane; `LoggingTunnel` probably should *not* start + logging REST traffic by default. Both need deciding rather than falling out. + +### One question this already answers + +Open question 5 below asked whether `HealthCheckContext` carries enough for a real probe. It does: +`IConnectionMultiplexer.RawConfig` and `ConfigurationOptions.Tunnel` are **both public**, so an +external probe can already reach `context.Server.Multiplexer.RawConfig.Tunnel` with no change to +`HealthCheckContext` at all. + +*Corrected 2026-10-08:* `RawConfig` is **not** public. The declaration found was on the internal +`IInternalConnectionMultiplexer` in the same file; the public `IConnectionMultiplexer` does not expose it. +So an external probe cannot reach the member's configured endpoints, which the lag-aware probe needs to +derive the REST endpoint and discover the database - one more reason it lives in this assembly. + +## 9. Open questions + +1. **Which tolerance default?** 100 ms (server, docs, redis-py) or 5000 ms (Lettuce)? Worth asking + Redis directly; the divergence looks unintentional. +2. **`Unhealthy` or `Inconclusive` when the REST call fails?** See §4. Recommend `Inconclusive`, but + note that this makes us behave differently from every other client, so it needs to be a stated + choice. +3. **Which endpoint — `/v1/bdbs/{uid}/availability` or `/v1/local/bdbs/{uid}/endpoint/availability`?** + Lettuce uses the former. The latter does not redirect to the primary node, which may be the more + accurate signal when probing a specific endpoint, and is what the docs recommend for load + balancers under the `all-nodes` proxy policy. Note known issue RS155734 against the endpoint form. + *Partly answered (§10):* the local form answers for whichever node served the REST call, so through + the cluster FQDN it is effectively random under `proxy_policy: single`. Not usable from a client + unless it addresses a specific node. +4. **Explicit bdb uid, discovery, or both?** Explicit avoids JSON entirely (§5). +5. ~~**Is `HealthCheckContext` sufficient?**~~ Answered in §8 — `RawConfig` and + `ConfigurationOptions.Tunnel` are both public, so a probe can reach the tunnel from the context it + already gets. What remains: if a single probe instance is ever shared across members it still + needs to map endpoint → REST configuration, which per-member health checks make unnecessary but do + not forbid. +6. **Credentials and rotation.** Lettuce takes a `Supplier` so credentials can + rotate; redis-py supports Basic plus mTLS; our own `ClusterRestClient` pins a CA because the + management certificate is self-signed per environment (§7). So the probe needs *at least* Basic + plus CA pinning, and should not bake in a static password. +7. ~~**How is this tested?**~~ Largely answered in §7: the tier exists, in #3191. What is genuinely + open is **sequencing and scope** — an end-to-end lag-aware test is downstream of an unmerged + 107-file PR, so either this waits for #3191 or it starts at the stubbed tier and the scenario test + follows. Also open: whether the deployments we have access to expose a lag-inducing effect at all + (ask `GetValidTriggersAsync` rather than assuming `network_latency`). +8. **Does any of this need `src/` changes at all?** On the evidence so far, possibly none: + `HealthCheckProbe` is externally subclassable, `RawConfig`/`Tunnel` are public, and the REST + plumbing precedent lives in the test tier. The `Tunnel` addition in §8 is the one thing that would + have to land in the core package — and only if we want the management plane to honour tunnels. + +## 10. Measured against real deployments (2026-09-18 to 2026-09-30) + +Everything here was observed, not read. Three AWS-template environments, all RS 8.0.22, three nodes per +cluster; the last one (`marcgravell-test-dc1342d5`) has two clusters, `c1` and `c2`, carrying two +Active-Active databases between them. Where a line is an inference rather than an observation it says so. + +### `extend_check=lag` is an Active-Active check + +Same cluster (c1), same session: + +| database | `crdt` | `replication` | plain | `extend_check=lag` at 100 ms / 600000 ms / 0 ms | +| --- | --- | --- | --- | --- | +| `re-active-active` | true | false | 200 | 200 / 200 / 200 | +| `re-active-active-oss-cluster` | true | false | 200 | 200 / 200 / 200 | +| control, freshly created | false | false | 200 | 503 / 503 / 503 | +| control, freshly created | false | true | 200 | 503 / 503 / 503 | + +Both Active-Active databases answer the same on c2. The control 503 is always +`{"error_code":"bdb_unavailable","description":"BDB is not available"}`, held for the whole 30 s +observed from creation, and was reproduced on two earlier environments over 60 s. + +- **Measured:** off Active-Active, the lag check fails regardless of replication and tolerance - the + tolerance parameter has no observable effect at all there. +- **Inferred:** the server has no sync lag to measure on such a database and reports "unavailable" + rather than "not applicable". +- **Consequence for the probe:** lag-awareness on a non-Active-Active member turns a healthy database + into a permanently unhealthy one. It must be opt-in per member, and that specific 503 should not be + read as a data-plane verdict - `Inconclusive` at most. This is a stronger version of the §4 argument. +- **Inferred about Lettuce, untested:** `EXTENDED_CHECK_DEFAULT = true` should make `LagAwareStrategy` + report `UNHEALTHY` forever against any database that is not Active-Active. Worth putting to Redis with + the 100 ms versus 5000 ms question (§3). +- **Unknown:** whether a 0 ms tolerance is honoured (and an idle link simply reports zero lag) or + ignored. The idle Active-Active result cannot tell those apart; it needs real lag. + +### Under real lag: the check flips, and only on the stale member (2026-10-05) + +Environment `marcgravell-test-9db7bbf0`, two clusters, `re-active-active` (uid 1 on both, paired by +`crdt_guid`). Sync *into* c2's member paused with `PUT /v1/bdbs/1 {"crdt_sync":"paused"}` on c2 for 60 s +while c1 took an `INCR` every ~5 s and c2 was read back; then `{"crdt_sync":"enabled"}`. + +| phase | c1 member | c2 member | data | +| --- | --- | --- | --- | +| baseline | plain 200, lag 200 (100 / 600000 / 0 ms) | plain 200, lag 200 (all three) | c2 tracks c1 | +| c2 sync paused | unchanged | plain 200, **lag 503 at all three**, first sample ~2 s in | c1 3 to 14, c2 stuck at 2 | +| resumed | unchanged | lag 503 ~5 s more, then 200 | c2 reaches 14 | + +- **Measured:** the plain check never noticed; the lag check on the stale member flipped within seconds + and cleared once the data had caught up. This is the "reachable but stale" case the feature exists for. +- **Measured:** only the member that is *behind* flips. The member being written to stays green - which + is the right polarity for failback, where the question is "is the target caught up". +- **Measured:** with sync paused, 600000 ms still answers 503 although the true lag was seconds. A paused + link is unavailable regardless of tolerance. +- **Measured: the tolerance does govern a running link.** Sync enabled; a server-side Lua loop on c1 + wrote ~50 MB per call (500 x 100 KB over a 200-key space) for 30 s, ~2.75 GB in all, while c2 was + sampled every ~0.5 s. In 66 samples the 100 ms check failed twice, each time with c2 visibly a few + batches behind, and recovered on the next sample; the 600000 ms check never failed. So 100 ms and + 600000 ms disagree exactly when the link is running but behind - the parameter is honoured. +- **Caveat on the numbers:** both clusters are in `us-east-1`. Under that load sync stayed under + 100 ms almost all the time, but that says nothing about a cross-region link, which is the case the + 100 ms versus 5000 ms question (§3) is actually about. +- **Measured:** the failure body is the same as for a non-Active-Active database - + `{"error_code":"bdb_unavailable","description":"BDB 1 is not available"}`. No lag-specific code, so a + probe cannot tell "lagging" from "not applicable" by the response; Active-Active membership has to be + configuration, not inference. +- **Inferred:** at baseline c2 was observed one write behind (read 1 after c1 wrote 2) while the 0 ms + check passed - either lag is sampled coarsely or 0 is not honoured. +- **Tooling:** `crdt_sync` `paused`/`enabled` is a narrow, reliably reversible way to produce lag - one + member, no network change, `status` returns to `active` within a few seconds. Far better suited to a + scenario test than `network_latency` or `network_failure` (which drops all inbound bar SSH on every + hosting node and kills `dmcproxy`). It is a direct REST mutation, not an injector action. + +### Under a partition: blind for about a minute, then tolerance-governed, and symmetric (2026-10-08) + +Environment `marcgravell-test-41f58c28`. A network partition between the clusters, nothing else: on each +c1 node, iptables rules tagged `lagprobe` dropped traffic to and from c2's six node addresses (private and +public), with a `systemd-run` timer on each node as a self-heal backstop. SSH, DNS, 9443 and the client +path were untouched. 60 s, then removed; all three nodes verified back at their baseline rule sets. +Writes to c1 every ~5 s; both members sampled (database and local forms). + +| observer time | c2 data | c2 lag check (100 / 600000 / 0 ms) | c1 lag check (100 / 600000 / 0 ms) | +| --- | --- | --- | --- | +| t+0 to 27 s, before | in step | 200 / 200 / 200 | 200 / 200 / 200 | +| t+28 s | **partition applied** | | | +| t+31 to 84 s | **stuck at 8**, c1 reaching 20 | **200 / 200 / 200** | 200 / 200 / 200 | +| t+88 s | stuck | **503 / 200 / 503** | 200 / 200 / **503** | +| t+94 s | stuck | 503 / 200 / 503 | **503** / 200 / 503 | +| t+92 s | **partition removed** | | | +| t+101 s on | caught up | 200 / 200 / 200 | 200 / 200 / 200 | + +Plain checks 200 throughout, both forms, both members; the local form tracked the database form. + +- **Measured: the lag check is blind to a dead link for about 60 s.** Sync stopped at once, but both + members reported no lag for ~57 s. Unlike paused sync, which fails immediately at any tolerance. +- **Measured: then it compares a growing lag against the tolerance.** 0 ms failed first, then 100 ms; + 600000 ms never failed. **A tolerance of 0 is honoured** (open since 2026-09-30). +- **Measured: both members fail.** Each is missing the other's stream, so "every member lagging" is the + ordinary state of a partition, not an edge case. +- **Inferred:** a dead peer region looks the same from the survivor as this partition does, so the + survivor's lag check passes for about a minute after the region dies and fails after that (at 100 ms). + The ~60 s presumably comes from the syncer's own failure detection; not investigated. + +### Other answers + +- **Cluster default** `availability_lag_tolerance_ms` is 100, on every environment - the server agrees + with the docs and redis-py, not Lettuce. +- **`Host: cnm.cluster.fqdn` is not required.** A bogus `Host` on the availability route still gets 200. +- **Both error-code families are real.** Database form: `bdb_unavailable`. Local form: + `bdb_endpoint_unavailable`, description `"Local endpoint of BDB on node is not available"`. + The composed suffixes (`_shard_unreachable`, `_port_unbound`) have not been observed yet - nothing so + far has broken a shard. +- **Healthy is an empty 200.** No body to parse; JSON appears only on failure, as §5 hoped. +- **`fields=` filters on `/v1/bdbs` but not on `/v1/cluster`**, which returns the whole object. +- **The local form answers for the node that served the request.** Through the cluster FQDN that was + always node 1, which usually does not host the endpoint under `proxy_policy: single` - so 503 for three + of four Active-Active members that were serving traffic perfectly well. *Corrected 2026-10-06:* reached + through the **database's own hostname** (`https://redis-.:9443`, as go-redis does), the + local form answered 200, plain and lag, for all four members, from the node actually serving the + endpoint; via the cluster FQDN the same calls gave 503 for three of them. Every node presents the same + cluster certificate (pinned on each), and its `*.` SAN covers the database hostname, so the + `TrustIssuer` pin should hold there too (*inferred*; only the fingerprint was checked on that path). So + the local form is usable, and arguably the more truthful one, provided it is routed by database host. + *Measured 2026-10-07 (`marcgravell-test-823d3f1e`):* under paused sync the local form, routed by + database host, behaves exactly like the database form - plain 200 throughout; lag 503 on the stale + member only, at 100 ms and 600000 ms alike, from the first sample (~2 s); cleared on the same sample as + the database form once data caught up. Its lag failure is `bdb_endpoint_unavailable`, which is also + what it answers on a node not serving the endpoint, so it is no more distinguishable by body than the + database form. For lag, the two forms give the same verdict; the difference is call shape (one per + database versus one per endpoint, and the local form only works routed by database host). + *Qualified 2026-10-08 (`marcgravell-test-41f58c28`):* routing by database host is fine for the + availability routes, but **not for the management routes**. On any node other than the master, + `/v1/bdbs?fields=...`, `/v1/bdbs/{uid}?fields=...` and `/v1/cluster` answer **307, redirecting to the + master's internal address** (`https://10.0.101.x:9443/...`), while `/availability` is answered directly. + Following that redirect from outside the VPC hangs. The cluster FQDN resolves to the master, which + answers everything. Found by running the real probe against the live clusters: three of four members + timed out until the probe derived the cluster name (strip `redis-{port}.`) instead and stopped following + redirects. *Inferred:* go-redis's discovery, routed via the database host, would meet the same redirect. +- **`/v1/bdbs` returns database passwords** (`authentication_redis_pass`, `authentication_admin_pass`) + when `mask_bdb_credentials` is false, as on these clusters, and `/v1/crdbs` returns the same for every + instance. Captured bodies cannot be committed as fixtures without scrubbing. Relevant to credential + scope too: whether `cluster_viewer`/`db_viewer` see the same was not checked (we used admin). + +### What a probe can discover, and with what (2026-10-06, `marcgravell-test-60790432`) + +**Active-Active membership is not visible over RESP.** Compared on c1: both Active-Active members and a +freshly created plain database (since deleted). + +- `HELLO` is identical in shape (`mode: cluster`, `role: master` on all three, including the plain one). +- `CRDT.INFO` is an unknown command everywhere; `INFO crdt` / `INFO crdb` are empty everywhere; no INFO + field mentions crdt. +- The only INFO differences are incidental: `cmdstat_replconf` on the members (the syncer's own traffic, + so absent until a peer has connected), a different module list (the members load only ReJSON and + search), and `rdb_changes_since_last_save`. None is a signal to build on. +- So the probe has to learn it from REST: `GET /v1/bdbs/{uid}?fields=uid,crdt` answers + `{"crdt":true,"uid":1}`, or it is configuration. + +**`db_viewer` is enough, and the least privileged role still sees database passwords.** Temporary +roles (`management: db_viewer`, then `cluster_viewer`) and users, deleted afterwards. RS 8 rejects the +legacy `role` field (`role_not_exist`); roles are objects referenced by `role_uids`, and a fresh cluster +has only `Admin`. + +| route, as `db_viewer` (`cluster_viewer` identical) | result | +| --- | --- | +| `/v1/bdbs/{uid}/availability`, with and without `extend_check=lag` | 200 | +| `/v1/bdbs/{uid}?fields=uid,crdt` | 200, no password fields | +| `/v1/bdbs?fields=uid,endpoints` | 200, 599 bytes, no password fields | +| `/v1/bdbs/{uid}` (unfiltered) | 200, **`authentication_redis_pass` and `authentication_admin_pass` populated** | +| `/v1/crdbs` | 200, every instance's passwords populated | + +This is with `mask_bdb_credentials: false`, the cluster's setting here; masked clusters were not +tested. Consequence: the probe must only ever issue `fields=`-filtered requests, and the docs should +recommend `db_viewer`. Possibly worth raising with the RS team that the viewer roles see secrets. + +**The REST endpoint is derivable from the database hostname.** Every database's `dns_name`, plain, +Active-Active and OSS-cluster alike, is `redis-.`, so the management endpoint is +`https://:9443`. But the OSS-cluster database advertises **IP addresses** through +`CLUSTER SLOTS`/`CLUSTER SHARDS` (`oss_cluster_api_preferred_endpoint_type: ip`), so the derivation must +start from the member's *configured* endpoint, not from discovered nodes. A customer CNAME or a bare IP +defeats it, so it can only be a default with an override. uid discovery by matching `dns_name` and port +against `/v1/bdbs?fields=uid,endpoints` is safe on credentials (filtered, no passwords). + +### Certificate trust reuses the existing TLS callback (2026-10-06) + +The library's own `ConfigurationOptions.TrustIssuer(...)` callback, obtained through the public +`TlsOptions.CertificateValidationCallback` and adapted to `HttpClientHandler` with a one-line lambda, +validates the 9443 call. Against c1, `GET /v1/cluster`: + +| trust | result | +| --- | --- | +| platform default | rejected, `UntrustedRoot` | +| `TrustIssuer(c1's own 9443 certificate)` | **200** | +| `TrustIssuer(config-dir ca.crt)` - the CA that signs the RESP proxy certs | rejected | +| `TrustIssuer(c2's 9443 certificate)` | rejected | + +The only policy error ever presented was `RemoteCertificateChainErrors`: the certificate carries SANs for +`` and `*.` and server-auth EKU, so host-name checks pass, which is the one condition +`TrustIssuer` requires. `TrustIssuer(leaf)` therefore *is* a pin, because the self-signed leaf is its own +chain. No new trust API is needed. + +What does **not** carry over is the trust *configuration*: the data-plane and management-plane +certificates have different issuers (proxy certs from the deployment CA, 9443 self-signed per cluster), +so a member's Redis `TrustIssuer` cannot be reused for its REST calls; it is the same mechanism with a +separate setting. The adapter is the same on every TFM (`ServerCertificateCustomValidationCallback`), and +on `net5`+ `SocketsHttpHandler.SslOptions.RemoteCertificateValidationCallback` takes the delegate +unchanged, which matters if the HTTP path rides on `Tunnel` (§8). A production cluster with a CA-signed +9443 certificate needs no callback at all. + +### Who else implements it, and what turns it on (2026-10-06) + +Read from anonymous shallow clones, heads of 2026-09-26 to 2026-10-06 (`gh` search is blocked by SSO). + +| client | support | activation | REST endpoint | uid | lag / tolerance | REST TLS | +| --- | --- | --- | --- | --- | --- | --- | +| Lettuce | `failover/health/LagAwareStrategy.java` | explicit strategy supplier; default `PingStrategy` | user-supplied URI | discovered from `/v1/bdbs` by host | on / 5000 | own `SslOptions` | +| Jedis | `mcf/LagAwareStrategy.java` | explicit strategy supplier; default `PingStrategy` | user-supplied | `/v1/bdbs?fields=uid,endpoints` by host | on / 5000 | own `SslOptions` | +| redis-py (asyncio only) | `asyncio/multidb/healthcheck.py` | explicit `health_checks`; default `PingHealthCheck` | user-supplied `health_check_url` + port 9443 | **unfiltered** `/v1/bdbs`, by `dns_name`/`addr` | always on / 5000 | own CA / `verify_tls` / mTLS | +| go-redis | **feature branch only** (`feature/multidb-integration`), `multidb/healthcheck_lag_aware.go` | explicit `HealthChecks`; default PING | **derived: `https://:9443`**, override available | `/v1/bdbs?fields=uid,endpoints`, host + resolved IPs + port | on / 5000 | own `tls.Config` incl. `InsecureSkipVerify` | +| node-redis, NRedisStack, redis-rs, rueidis, predis, phpredis | not found | | | | | | + +- **Nobody activates it automatically** - no domain detection, no provider. It is an opt-in health + check everywhere, and PING is the default everywhere. +- **No client checks `crdt`**, so (*inferred*, from §10) all fail permanently on non-Active-Active. +- **No client reuses the Redis connection's TLS settings** for REST, and none pins. +- redis-py's unfiltered `/v1/bdbs` receives database passwords (per the role test above). +- **go-redis has the two design ideas worth taking:** + 1. it uses the *local* form routed via each member's own database host, once per master in cluster + mode - the routing that makes the local form truthful (see the correction above); + 2. `FailbackOnly() == true`: the lag verdict only gates moving traffic *to* a member and never evicts + the active one, while PING keeps judging the active member's liveness. That is the `Inconclusive` + failback problem in `plan.md` step 3, solved by giving the check a scope rather than a new result. + +### Multi-cluster shape + +- `env_output.json` nests clusters under `.clusters.value[N]`, each with its own FQDN and credentials. + `FaultInjectorEnvironment` reads only `[0]`. +- Each cluster has its own self-signed 9443 certificate, so pins are per host + (`cluster-cert..sha256`). +- Member uids happened to match across c1 and c2; the platform does not promise it. `/v1/crdbs` on one + cluster reports the other's instance with `db_uid=None`, so pairing members means reading each + cluster's `/v1/bdbs` and matching on `crdt_guid`. The probe's configuration should therefore be + "cluster REST endpoint plus uid" per member, not a uid shared across the group. + +### Fault injection + +- **`network_latency` is not safe to use for this.** It applies `netem` to the node's primary interface, + not to the database, and its scheduled removal fails ("No existing session" - the cleanup thread reuses + an SSH session that has been torn down) while the action still reports success. On 2026-09-18 that + left a 5 s delay on two nodes, which took down the cluster's own DNS and 9443 until removed by hand + over SSH. Reported to `redis-developer/cae-client-testing`. +- **Holding back sync:** done with `crdt_sync=paused` on the stale member (above), not with the injector. diff --git a/notes/lag-aware/plan.md b/notes/lag-aware/plan.md new file mode 100644 index 000000000..9faef4b75 --- /dev/null +++ b/notes/lag-aware/plan.md @@ -0,0 +1,204 @@ +# Lag-aware availability checks: the plan + +What we intend to do about [`findings.md`](findings.md). That file is the evidence; this one is the +sequence. + +Status: **proposed. Step 0 is cleared; step 1 is answered (findings §10): the lag check flips under +real Active-Active lag, on the stale member only, and the tolerance is honoured on a running link. +Cross-region lag magnitudes remain unmeasured (both test clusters share a region).** The core change +(failback health check, member-scoped probes, selection rules) and the REST probe are implemented, with +stubbed-transport tests (step 5, tier 1) and a fault-injector scenario test (tier 3, +`LagAwareFailbackScenarioTests`, passing against a live Active-Active pair on 2026-10-08). Tier 2 (the +Kestrel toy) is not done and arguably not needed now that tiers 1 and 3 exist. +See [`api-sketch.md`](api-sketch.md). + +> **This branch is not only a planning branch.** `marc/lag-aware-availability` carries these notes +> now, and is the feature branch from here — the implementation lands on top of the same branch and +> the same PR, rather than the notes being merged separately and the work starting again elsewhere. +> +> It has been rebased onto `main` post-#3191 (`git rebase origin/main`), which is what step 0 asked +> for; do not merge `main` in. #3191 squash-merged, so merging it back would move the merge base +> without giving us its ancestry, and the commit list here would start carrying phantoms. + +> **Where this can be worked on.** Step 1 needs a real Redis Enterprise deployment and the fault +> injector, which live on a machine holding the environment directory (`~/aws` on the box that has +> it — the AWS multi-cluster template, so `env_output.json` nests its outputs under +> `.clusters.value[0]`, which `FaultInjectorEnvironment` already handles). Nothing in step 1 can be +> done or verified without it, so the work continues there rather than on whichever box happens to +> have the repo. Steps 2 onward are ordinary code and are not tied to a machine. + +## Step 0 — wait for #3191 ✅ cleared + +**Done.** #3191 merged as `dc915bbc` on 2026-09-18, and this branch has been rebased onto it, so +`tests/StackExchange.Redis.FaultInjector.Tests` is present here now. The rest of this section records +why the wait was worth it. + +[#3191](https://github.com/StackExchange/StackExchange.Redis/pull/3191) ("Server-native maintenance +notifications") is open, 107 files, +13,160. It carries +`tests/StackExchange.Redis.FaultInjector.Tests`, and with it the three things this work would +otherwise have to build from scratch: + +| what | why it matters here | +| --- | --- | +| `FaultInjector/FaultInjectorClient.cs` | `POST /action` + `GET /action/{id}`, plus `GetValidTriggersAsync(scenario, effect, …)` — lets us ask a deployment which lag-inducing effects it supports instead of guessing | +| `Environment/ClusterRestClient.cs` | already a Redis Enterprise REST client on 9443: Basic auth, CA pinning, `System.Text.Json`, `GET /v1/bdbs?fields=…` | +| the tier itself | environment fixtures, provisioning, gating (`SER_FI_CONFIG_DIR`, `E2E_SCENARIO_TESTS`), and the README describing how to run it | + +Building any of that again on this branch would mean writing a second copy of a REST client against +the same cluster, and then reconciling them at merge. Not worth it for a feature whose entire +implementation is smaller than the client that tests it. + +It has landed, so nothing here is gated on it any more. Step 1 is the next thing, and its only +prerequisite is a cluster to point at. + +## Step 1 — validate the API against a real deployment + +Before any code. Everything in findings §1 is read off documentation, and #3191's own summary is the +warning: *"the specifications are prose, the payloads were never published, and nearly every +assumption I started with was wrong in some way that mattered."* Same vendor, same class of API. + +The checklist, each item currently an assumption: + +- [x] does the deployed version support `extend_check=lag` at all? **Yes (RS 8.0.22), but only + meaningfully on Active-Active members; everything else gets `503 bdb_unavailable` + unconditionally.** Findings §10. +- [ ] does `availability_lag_tolerance_ms` as a query parameter genuinely override the cluster + default, and what happens if it is absurd (0, negative, enormous)? **Honoured on a running + link:** under a write burst 100 ms failed while 600000 ms passed. No effect off Active-Active or + with sync paused (both always fail). Negative not tried. +- [ ] what is actually in a failure body — is `error_code` reliably present, and are the composed + forms (`bdb_unavailable_shard_unreachable_port_unbound`) real? *Partly:* `error_code` and + `description` present on every failure seen; lag failure is plain `bdb_unavailable`, the same + as "not Active-Active"; composed suffixes not yet observed. +- [x] does the lag check flip while the plain check stays green? **Yes, on the stale member only, + within seconds, clearing when data catches up** (findings §10, `crdt_sync=paused`). +- [x] is `Host: cnm.cluster.fqdn` required on the availability route? **No.** +- [x] `/v1/bdbs/{uid}/availability` versus `/v1/local/bdbs/{uid}/endpoint/availability`? **The local + form reports for whichever node answered the REST call; unusable through the cluster FQDN.** Use + the database form. RS155734 not assessed. +- [ ] what does `GET /v1/bdbs?fields=uid,endpoints` return, in the shape Lettuce matches hosts + against? Not yet asked with those fields; full `/v1/bdbs` bodies captured (contain passwords, + not committed). +- [ ] **what lag do real geo-replicated links show under load?** The empirical way to settle 100 ms + (server default, docs, redis-py) versus 5000 ms (Lettuce) — see findings §3. Needs a two-cluster + environment, which now exists. +- [ ] which fault-injector effects can drive lag past the tolerance — ask `GetValidTriggersAsync`, + do not assume `network_latency`. **`network_latency` is ruled out** (node-wide, and its cleanup + fails; findings §10). The `/action` enum also lists `network_failure`, `execute_rladmin_command` + and `update_cluster_config`; which of these can hold back Active-Active sync safely is the open + question. + +Output: the answers written into `findings.md`, and **captured response bodies saved as fixtures**, so +the stubbed tests in step 5 replay real payloads instead of guessing at the wire format. + +## Step 2 — decide where it lives + +Findings §6 sets out three options; step 1 informs the choice, so this is deliberately not decided +here. The honest current state: + +- **The dependency argument is weaker than it first looked.** `System.Net.Http` needs no package + reference on any target we ship — it is in-box on .NET Framework, in `netstandard2.0`, and in the + shared framework on `net8.0`/`net10.0`. The genuinely new dependency would be `System.Text.Json` on + `net472`/`netstandard2.0`, and findings §5 shows JSON is avoidable if the bdb uid is configured + rather than discovered. +- **`src/` may need no changes at all** *(no longer true - see the `Inconclusive` point in step 3)*. `HealthCheckProbe` is externally subclassable (no internal + abstract members), and `IConnectionMultiplexer.RawConfig` and `ConfigurationOptions.Tunnel` are both + public, so a probe can reach everything it needs from the context it already gets. +- So the question is not really "can we" but "should a general-purpose Redis client carry a + management-plane HTTP client for one commercial deployment target" — which is a judgement call, not + a technical constraint. + +Leaning: **a separate package** (option B), `net8.0`+ only. It keeps Enterprise-specific surface out +of core, versions independently, and `SocketsHttpHandler.ConnectCallback` — needed for step 4 — is +net5+ anyway, so a down-level build would be degraded regardless. But a new shipping package is a real +ownership cost for a small feature, and that is mgravell's call, not this document's. + +## Step 3 — the probe + +```csharp +// shape only; names and home package per step 2 +public sealed class LagAwareHealthCheckProbe : HealthCheckProbe +{ + public override Task CheckHealthAsync(HealthCheckContext context); +} +``` + +Behaviour, with the deliberate deviations called out: + +- `GET /v1/bdbs/{uid}/availability`, adding `extend_check=lag` and `availability_lag_tolerance_ms` + when lag-awareness is on. +- 200 → `Healthy`. A definite negative (503 with a recognised `error_code`) → `Unhealthy`. +- **Lag-awareness is per member and only for Active-Active members.** Off Active-Active the server + answers `503 bdb_unavailable` to every lag check (findings §10), so enabling it there would fail a + healthy database forever. Either refuse the combination at configuration time, or check `crdt` once + via `/v1/bdbs/{uid}` and fall back to the plain check - the latter needs JSON, the former does not. +- Addressing is REST endpoint **plus** uid per member; uids are not stable across the clusters of one + Active-Active database. +- **The probe is called per endpoint, but the answer is per database.** `HealthCheck.CheckHealthAsync + (IConnectionMultiplexer)` fans out one probe call per server (`HealthCheck.Execute.cs:25-40`), so an + OSS-cluster member with six endpoints would make six identical REST calls per pass. Either cache + within a pass, or hook in at member level instead. +- **`Inconclusive` is right for failover and wrong for failback.** `ConnectionGroupMember.UpdateState` + (`MultiGroupMultiplexer.cs:315`) treats anything but `Unhealthy` as connected, and once + `FailbackDelay` has passed that makes the member eligible again. So "the REST API could not be + asked" would let the group fail back onto a member that may be stale - the exact case the lag check + exists to prevent. Keeping an active member while refusing to *return* to an unverified one needs a + distinction `UpdateState` does not currently make, which means a small `src/` change after all (step 2 + assumed none). +- **A failed REST call → `Inconclusive`, not `Unhealthy`.** This is the deviation from every other + client (findings §4): Lettuce's two-state `HealthStatus` cannot distinguish "the database is down" + from "I could not ask", so a management-plane outage can fail over a database that is serving + traffic perfectly. We have `Inconclusive`; we should use it. Document it as a difference. +- Configuration: REST endpoint, credentials **via a callback so they can rotate** (Lettuce takes a + `Supplier`), certificate options (step 1 confirmed the management certificate is self-signed by the + cluster itself, so neither ambient trust nor a separate CA works; the option has to accept a pinned + certificate or fingerprint), bdb uid, tolerance, and a lag-aware on/off switch. +- Defaults deferred to step 1's measurements rather than copied from either existing client. + +Explicit uid first; host-based discovery is a later, optional extra (findings §5). + +## Step 4 — the transport seam, if we want it + +Only if we want the management plane to honour tunnels. mgravell has offered a `Tunnel` addition; +findings §8 has the shape — a **stream-returning** member, composed into +`SocketsHttpHandler.ConnectCallback` on the consuming side, so no `System.Net.Http` type enters +StackExchange.Redis's public API. + +Two reasons it is worth doing beyond testing: a CONNECT proxy configured for Redis almost certainly +applies to the Enterprise REST API too, and `InProcessTestServer` already assigns `Tunnel`, so one +assignment would spoof both planes. Two things it forces: `HttpProxyTunnel` and `LoggingTunnel` each +need an opinion about whether they apply to the management plane. + +Separable from step 3 — the probe works without it, just without tunnel support. + +## Step 5 — tests + +Three tiers, cheapest first: + +1. **Stubbed transport.** Hijack the HTTP transport completely, the way `Tunnel` does for RESP, and + replay the step-1 fixtures: 200, each 503 `error_code`, 404, 401/403, malformed bodies, and + slow-but-successful responses for probe-timeout behaviour. No listener, no port, runs in CI. +2. **`toys/KestrelRedisServer`.** It already serves HTTP on 5000 over the same `RedisServer` + singleton that serves RESP, so a fake `/v1/bdbs/{uid}/availability` can lie *coherently with* the + data plane. Real sockets on both planes; the level where the whole thing is exercised. +3. **Fault-injector scenario test**, in the tier from step 0: drive real lag past the tolerance and + assert the lag-aware check flips while the plain check stays green — then that the group fails + over, and fails back only once lag recovers. Produce the lag with `crdt_sync=paused` on the target + member via its cluster's REST API (findings §10), and resume in a `finally`; not with + `network_latency`. + +Tier 3 is the authoritative one and the slowest; tier 1 is what runs on every push. + +## Step 6 — documentation + +`docs/` needs the probe, the tolerance decision and its reasoning, the credential/CA requirements, +and — prominently — the `Inconclusive` deviation, because anyone comparing us against Lettuce or +redis-py will otherwise read it as a bug. + +## Not doing + +- **Not writing a second Redis Enterprise REST client.** Step 0 exists so we extend + `ClusterRestClient` instead. +- **Not copying a tolerance default.** 100 ms and 5000 ms cannot both be right; step 1 measures. +- **Not implementing host-based discovery first.** Explicit uid keeps JSON out of the required path. +- **Not changing the profiling or health-check abstractions.** They already fit; that is the finding. diff --git a/notes/readme.md b/notes/readme.md new file mode 100644 index 000000000..b51960340 --- /dev/null +++ b/notes/readme.md @@ -0,0 +1,24 @@ +# Working notes + +This directory holds working material for whoever picks a topic up next: investigations, the +reasoning behind a decision, and snapshots that were expensive to produce and would be expensive +to produce again. + +The split to keep: + +- `docs/` — documentation for people *using* StackExchange.Redis. It is a Jekyll site + (`docs/_config.yml`, published at ), so anything placed there is built into + the site *and* listed in `sitemap.xml`. +- `notes/` — internal handover material. Not built, not published, not indexed. + +Notes are grouped by topic in a subdirectory, and the files inside are named for their content +rather than repeating the topic — `notes/lag-aware/findings.md`, not +`notes/lag-aware/lag-aware-findings.md`. + +When something in here graduates into advice for users, write it up in `docs/` and leave the note +as the record of *why*. + +## Topics + +- [`lag-aware/`](lag-aware/) — Redis Enterprise lag-aware availability checks, and whether they + belong in the geo-redundant failover health checks. diff --git a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs index 1ee5dda8a..9869d63a9 100644 --- a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs @@ -20,7 +20,15 @@ private async Task CheckHealthCoreAsync(IConnectionMultiplexe try { Task[] pending; - if (multiplexer is IInternalConnectionMultiplexer internalMultiplexer) + if (Probe.Scope is HealthCheckProbeScope.Member) + { + // the answer is about the database, not an endpoint: ask once, via any connected server + var server = FirstConnectedServer(multiplexer); + if (server is null) return HealthCheckResult.Unhealthy; + pending = GetReusablePending(ref _reusablePending, 1); + pending[0] = CheckHealthAsync(server); + } + else if (multiplexer is IInternalConnectionMultiplexer internalMultiplexer) { var snapshot = internalMultiplexer.GetServerSnapshot(); pending = GetReusablePending(ref _reusablePending, snapshot.Length); @@ -51,6 +59,26 @@ private async Task CheckHealthCoreAsync(IConnectionMultiplexe } } + private static IServer? FirstConnectedServer(IConnectionMultiplexer multiplexer) + { + if (multiplexer is IInternalConnectionMultiplexer internalMultiplexer) + { + foreach (var endpoint in internalMultiplexer.GetServerSnapshot()) + { + if (endpoint.IsConnected) return endpoint.GetRedisServer(null); + } + + return null; + } + + foreach (var server in multiplexer.GetServers()) + { + if (server.IsConnected) return server; + } + + return null; + } + internal int TotalTimeoutMillis() { bool valid = TryComputeTotalTimeoutMillis(ProbeCount, ProbeTimeout, ProbeInterval, out int millis); diff --git a/src/StackExchange.Redis/Availability/HealthCheck.LagAware.cs b/src/StackExchange.Redis/Availability/HealthCheck.LagAware.cs new file mode 100644 index 000000000..5ccbad622 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheck.LagAware.cs @@ -0,0 +1,16 @@ +namespace StackExchange.Redis.Availability; + +public sealed partial class HealthCheck +{ + /// + /// A health check that asks the Redis Enterprise database availability API, once per member and once per + /// pass, whether the database is available and (for an Active-Active member) caught up; intended as a + /// . See . + /// + /// + /// For different probe counts or timeouts, use + /// new HealthCheck.Builder { Probe = HealthCheckProbe.LagAware(options), ... }. + /// + public static HealthCheck LagAware(LagAwareOptions options) + => new Builder { Probe = HealthCheckProbe.LagAware(options), ProbeCount = 1 }.Create(); +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.LagAware.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.LagAware.cs new file mode 100644 index 000000000..2546df4f6 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.LagAware.cs @@ -0,0 +1,249 @@ +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Ask the Redis Enterprise database availability API whether the member's database is available and, + /// for an Active-Active member, caught up; see . Runs once per member. + /// + /// + /// Healthy on HTTP 200; unhealthy on 503 (unavailable or lagging) or 404 (no such database); + /// when the question could not be asked (the REST API + /// unreachable, credentials rejected, the database not found by discovery), so that a management-plane + /// problem is never mistaken for the database being down. Intended as a + /// . + /// + public static HealthCheckProbe LagAware(LagAwareOptions options) => new LagAwareProbe(options); + + internal sealed class LagAwareProbe : HealthCheckProbe + { + private const int DefaultRestPort = 9443; + + private readonly Func> _credentials; + private readonly Uri? _restEndpoint; + private readonly int? _databaseId; + private readonly LagCheckMode _mode; + private readonly TimeSpan? _tolerance; + private readonly HttpClient _http; + + // per (REST endpoint, database endpoint): one probe can serve every member of a group + private readonly ConcurrentDictionary _discovered = new(StringComparer.OrdinalIgnoreCase); + + private sealed class Discovered + { + public int? Uid; + public bool? ActiveActive; + } + + public LagAwareProbe(LagAwareOptions options) + { + if (options is null) throw new ArgumentNullException(nameof(options)); + _credentials = options.Credentials ?? throw new ArgumentException("Credentials are required.", nameof(options)); + if (options.LagTolerance is { } tolerance && tolerance < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(options), tolerance, "LagTolerance must not be negative."); + } + + _restEndpoint = options.RestEndpoint; + _databaseId = options.DatabaseId; + _mode = options.LagCheck; + _tolerance = options.LagTolerance; + var handler = options.HttpMessageHandlerFactory?.Invoke() ?? CreateHandler(options.CertificateValidation, options.ClientCertificate); + _http = new HttpClient(handler) { Timeout = Timeout.InfiniteTimeSpan }; // bounded per call instead + } + + public override HealthCheckProbeScope Scope => HealthCheckProbeScope.Member; + + internal static HttpMessageHandler CreateHandler(RemoteCertificateValidationCallback? validation, X509Certificate2? clientCertificate) + { +#if NET + // redirects point at a node's internal address, so following one only ever hangs; report it instead + var handler = new SocketsHttpHandler { PooledConnectionLifetime = TimeSpan.FromMinutes(2), AllowAutoRedirect = false }; + if (validation is not null) handler.SslOptions.RemoteCertificateValidationCallback = validation; + if (clientCertificate is not null) handler.SslOptions.ClientCertificates = [clientCertificate]; + return handler; +#elif NET461 + if (validation is not null || clientCertificate is not null) + { + throw new PlatformNotSupportedException("Custom certificate validation and client certificates for the REST API require .NET Framework 4.7.1 or later."); + } + return new HttpClientHandler { AllowAutoRedirect = false }; +#else + var handler = new HttpClientHandler { AllowAutoRedirect = false }; + if (validation is not null) + { + handler.ServerCertificateCustomValidationCallback = (request, certificate, chain, errors) => validation(request, certificate, chain, errors); + } + if (clientCertificate is not null) + { + handler.ClientCertificateOptions = ClientCertificateOption.Manual; + handler.ClientCertificates.Add(clientCertificate); + } + return handler; +#endif + } + + public override async Task CheckHealthAsync(HealthCheckContext context) + { + var multiplexer = context.Server.Multiplexer; + var endpoints = (multiplexer as IInternalConnectionMultiplexer)?.RawConfig.EndPoints; + var configured = endpoints is { Count: > 0 } ? endpoints[0] : null; + if (!Format.TryGetHostPort(configured, out var host, out var port)) + { + return Inconclusive(multiplexer, "(none)", "the member has no configured endpoint"); + } + + var rest = _restEndpoint ?? new UriBuilder(Uri.UriSchemeHttps, ClusterHost(host, port.Value), DefaultRestPort).Uri; + var key = $"{rest}|{host}:{port}"; + var known = _discovered.GetOrAdd(key, static _ => new Discovered()); + + // finish inside the probe timeout, so that "could not ask" is reported as such rather than as the + // Unhealthy a timed-out probe is otherwise given + using var cts = new CancellationTokenSource(Budget(context.ProbeTimeout)); + try + { + var credential = await _credentials(cts.Token).ConfigureAwait(false); + var auth = new AuthenticationHeaderValue( + "Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credential.UserName}:{credential.Password}"))); + + int? uid = _databaseId ?? known.Uid; + if (uid is null) + { + // filtered: an unfiltered /v1/bdbs carries database passwords, even for db_viewer + using var list = await GetAsync(rest, "v1/bdbs?fields=uid,endpoints", auth, cts.Token).ForAwait(); + if (list.StatusCode != HttpStatusCode.OK) return Inconclusive(multiplexer, rest, Describe("database discovery", list)); + uid = FindUid(await list.Content.ReadAsStringAsync().ForAwait(), host, port!.Value); + if (uid is null) return Inconclusive(multiplexer, rest, $"no database on this cluster has the endpoint {host}:{port}"); + known.Uid = uid; + } + + bool lag = _mode switch + { + LagCheckMode.Enabled => true, + LagCheckMode.Disabled => false, + _ => known.ActiveActive ?? false, + }; + if (_mode is LagCheckMode.Auto && known.ActiveActive is null) + { + using var bdb = await GetAsync(rest, $"v1/bdbs/{uid}?fields=uid,crdt", auth, cts.Token).ForAwait(); + if (bdb.StatusCode == HttpStatusCode.NotFound) return Forget(key); + if (bdb.StatusCode != HttpStatusCode.OK) return Inconclusive(multiplexer, rest, Describe("database lookup", bdb)); + lag = IsActiveActive(await bdb.Content.ReadAsStringAsync().ForAwait()); + known.ActiveActive = lag; + } + + var path = $"v1/bdbs/{uid}/availability"; + if (lag) + { + path += "?extend_check=lag"; + if (_tolerance is { } tolerance) path += $"&availability_lag_tolerance_ms={(long)tolerance.TotalMilliseconds}"; + } + + using var response = await GetAsync(rest, path, auth, cts.Token).ForAwait(); + return response.StatusCode switch + { + HttpStatusCode.OK => HealthCheckResult.Healthy, + HttpStatusCode.ServiceUnavailable => HealthCheckResult.Unhealthy, // unavailable, or lagging + HttpStatusCode.NotFound => Forget(key), + _ => Inconclusive(multiplexer, rest, Describe("availability", response)), + }; + } + catch (Exception ex) when (ex is HttpRequestException or OperationCanceledException or JsonException or IOException) + { + return Inconclusive(multiplexer, rest, ex is OperationCanceledException ? "timed out" : ex.Message); + } + } + + // Database endpoints are named "redis-{port}.{cluster fqdn}"; ask the cluster name, not the database's. Any + // node answers the availability routes itself, but the others (discovery, the crdt lookup) are answered + // only by the master: other nodes redirect to its *internal* address, unreachable from outside the + // cluster's network. The cluster name resolves to the master. A host that does not follow the pattern + // is used as given. + internal static string ClusterHost(string host, int port) + { + var prefix = $"redis-{port}."; + return host.Length > prefix.Length && host.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + ? host.Substring(prefix.Length) + : host; + } + + // the database has gone (or the configured uid is wrong): unhealthy, and rediscover next time + private HealthCheckResult Forget(string key) + { + _discovered.TryRemove(key, out _); + return HealthCheckResult.Unhealthy; + } + + private static string Describe(string what, HttpResponseMessage response) + => response.Headers.Location is { } location + ? $"{what} answered {(int)response.StatusCode}, redirecting to {location} (set RestEndpoint to an address that answers directly)" + : $"{what} answered {(int)response.StatusCode}"; + + private static TimeSpan Budget(TimeSpan probeTimeout) + { + var budget = TimeSpan.FromTicks(probeTimeout.Ticks / 10 * 9); + return budget > TimeSpan.Zero ? budget : TimeSpan.FromMilliseconds(1); + } + + private async Task GetAsync(Uri rest, string path, AuthenticationHeaderValue auth, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(rest, path)); + request.Headers.Authorization = auth; + return await _http.SendAsync(request, cancellationToken).ForAwait(); + } + + private static HealthCheckResult Inconclusive(IConnectionMultiplexer multiplexer, object rest, string reason) + { + (multiplexer as ConnectionMultiplexer)?.Logger?.LogWarningLagAwareInconclusive(rest.ToString() ?? "", reason); + return HealthCheckResult.Inconclusive; + } + + // GET /v1/bdbs?fields=uid,endpoints => [{"uid":1,"endpoints":[{"dns_name":"redis-1.example","port":1,"addr":["10.0.0.1"]}]}] + internal static int? FindUid(string json, string host, int port) + { + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind != JsonValueKind.Array) return null; + foreach (var bdb in document.RootElement.EnumerateArray()) + { + if (!bdb.TryGetProperty("uid", out var uid) || !uid.TryGetInt32(out var id)) continue; + if (!bdb.TryGetProperty("endpoints", out var endpoints) || endpoints.ValueKind != JsonValueKind.Array) continue; + foreach (var endpoint in endpoints.EnumerateArray()) + { + if (!endpoint.TryGetProperty("port", out var p) || !p.TryGetInt32(out var endpointPort) || endpointPort != port) continue; + if (endpoint.TryGetProperty("dns_name", out var dns) && string.Equals(dns.GetString(), host, StringComparison.OrdinalIgnoreCase)) return id; + if (endpoint.TryGetProperty("addr", out var addrs) && addrs.ValueKind == JsonValueKind.Array) + { + foreach (var addr in addrs.EnumerateArray()) + { + if (string.Equals(addr.GetString(), host, StringComparison.OrdinalIgnoreCase)) return id; + } + } + } + } + + return null; + } + + // GET /v1/bdbs/{uid}?fields=uid,crdt => {"crdt":true,"uid":1} + internal static bool IsActiveActive(string json) + { + using var document = JsonDocument.Parse(json); + return document.RootElement.ValueKind == JsonValueKind.Object + && document.RootElement.TryGetProperty("crdt", out var crdt) + && crdt.ValueKind == JsonValueKind.True; + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs index 6b449e034..6edb08208 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbe.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs @@ -15,6 +15,11 @@ public abstract partial class HealthCheckProbe /// public abstract Task CheckHealthAsync(HealthCheckContext context); + /// + /// Whether this probe is invoked per endpoint (the default) or once per member. + /// + public virtual HealthCheckProbeScope Scope => HealthCheckProbeScope.Endpoint; + private static Task? _inconclusive, _healthy, _unhealthy; /// diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs index b09848814..19e3869d3 100644 --- a/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs +++ b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs @@ -52,8 +52,10 @@ public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) // Fail as soon as we have any failure if (context.Failure > 0) return HealthCheckResult.Unhealthy; - // Succeed only when all probes have succeeded (no remaining) - if (context.Remaining == 0) return HealthCheckResult.Healthy; + // With no failures and nothing remaining, succeed - but only if something actually succeeded: a run + // of nothing but inconclusive probes ("could not tell") is not a pass, and a failback check + // depends on the difference + if (context.Remaining == 0) return context.Success > 0 ? HealthCheckResult.Healthy : HealthCheckResult.Inconclusive; // Can't determine yet return HealthCheckResult.Inconclusive; @@ -70,8 +72,10 @@ public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) // Succeed as soon as we have any success if (context.Success > 0) return HealthCheckResult.Healthy; - // Fail only when all probes have failed (no remaining) - if (context.Remaining == 0) return HealthCheckResult.Unhealthy; + // With nothing remaining and no success, fail - but only if something actually failed: a run of + // nothing but inconclusive probes ("could not tell") is not a failure, and reporting it as one marks + // a member unhealthy that may be perfectly well (a key-write probe on a replica, for example) + if (context.Remaining == 0) return context.Failure > 0 ? HealthCheckResult.Unhealthy : HealthCheckResult.Inconclusive; // Can't determine yet return HealthCheckResult.Inconclusive; diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbeScope.cs b/src/StackExchange.Redis/Availability/HealthCheckProbeScope.cs new file mode 100644 index 000000000..77ce912d1 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbeScope.cs @@ -0,0 +1,19 @@ +namespace StackExchange.Redis.Availability; + +/// +/// How often a is invoked when a whole member is checked. +/// +public enum HealthCheckProbeScope +{ + /// + /// Once per server endpoint of the member; the default, and right for probes that test the connection + /// itself (such as ). + /// + Endpoint = 0, + + /// + /// Once per member, against one connected endpoint; for probes whose answer is about the database as a + /// whole rather than any single endpoint, such as a call to a management API. + /// + Member = 1, +} diff --git a/src/StackExchange.Redis/Availability/LagAwareOptions.cs b/src/StackExchange.Redis/Availability/LagAwareOptions.cs new file mode 100644 index 000000000..e74b4f08d --- /dev/null +++ b/src/StackExchange.Redis/Availability/LagAwareOptions.cs @@ -0,0 +1,135 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +/// +/// Configures a health check against the Redis Enterprise database availability API, optionally asking +/// whether the database has caught up with its Active-Active peers; see . +/// +/// +/// Only is required. The REST endpoint, the database id and whether the database is +/// an Active-Active member are discovered, and no lag tolerance is sent unless one is set, so the cluster's +/// own setting applies. The options are read when the probe is created; later changes have no effect on it. +/// +public sealed class LagAwareOptions +{ + /// + /// Supplies the cluster management credentials, called for every request so that they can rotate. + /// + /// + /// These are the cluster's REST API credentials, not the database's. A role with + /// management: db_viewer is sufficient. + /// + public Func>? Credentials { get; set; } + + /// + /// The cluster REST API to ask; when (the default), derived from the member's first + /// configured endpoint: https://{cluster}:9443/ for a database named redis-{port}.{cluster} (the + /// form Redis Enterprise assigns), else https://{host}:9443/. + /// + /// + /// The cluster name matters: only the cluster's master answers every route the probe uses, and other nodes + /// redirect to its internal address. Redirects are never followed; set this explicitly if the database is + /// reached through a name that does not follow the pattern. + /// + public Uri? RestEndpoint { get; set; } + + /// + /// The database id (uid) on that cluster; when (the default), it is found by + /// matching the member's first configured endpoint against the cluster's databases. + /// + public int? DatabaseId { get; set; } + + /// + /// Whether to ask the lag question; (the default) asks it only of + /// Active-Active databases, since the server reports any other database as unavailable when asked. + /// + public LagCheckMode LagCheck { get; set; } + + /// + /// The lag tolerance to request; when (the default), none is sent and the cluster's + /// own availability_lag_tolerance_ms applies. + /// + public TimeSpan? LagTolerance { get; set; } + + /// + /// Validates the REST API's certificate; when , the platform's default validation + /// applies. + /// + /// + /// The management certificate of a Redis Enterprise cluster is self-signed by the cluster unless one has + /// been installed, so this is usually needed; see . Not supported on .NET + /// Framework 4.6.1. + /// + public RemoteCertificateValidationCallback? CertificateValidation { get; set; } + + /// + /// Trust certificates issued by the given certificate (which may be the cluster's own self-signed + /// certificate), as does for the data plane. + /// + public void TrustIssuer(string issuerCertificatePath) + => CertificateValidation = ConfigurationOptions.TrustIssuerCallback(issuerCertificatePath); + + /// + /// Trust certificates issued by the given certificate (which may be the cluster's own self-signed + /// certificate), as does for the data plane. + /// + public void TrustIssuer(X509Certificate2 issuer) + => CertificateValidation = ConfigurationOptions.TrustIssuerCallback(issuer); + + /// + /// A client certificate to present to the REST API, for clusters that require mutual TLS on it; when + /// (the default), none. Not supported on .NET Framework 4.6.1. + /// + public X509Certificate2? ClientCertificate { get; set; } + +#if NET + /// + /// Present a client certificate loaded from a PEM file pair, as + /// does for the data plane. + /// + /// The path for the user certificate (commonly a .crt file). + /// The path for the user key (commonly a .key file). + public void SetUserPemCertificate(string userCertificatePath, string? userKeyPath = null) + => ClientCertificate = ConfigurationOptions.LoadPemUserCertificate(userCertificatePath, userKeyPath); +#endif + + /// + /// Present a client certificate loaded from a PFX file, as + /// does for the data plane. + /// + /// The path for the user certificate (commonly a .pfx file). + /// The password for the certificate file. + public void SetUserPfxCertificate(string userCertificatePath, string? password = null) + => ClientCertificate = ConfigurationOptions.LoadPfxUserCertificate(userCertificatePath, password); + + // test seam: replaces the HTTP transport entirely, so tests can answer requests without a listener + internal Func? HttpMessageHandlerFactory { get; set; } +} + +/// +/// Whether a check asks the lag question. +/// +public enum LagCheckMode +{ + /// + /// Ask only if the database is an Active-Active member; otherwise check plain availability. + /// + Auto = 0, + + /// + /// Always ask. A database that is not an Active-Active member is then always reported unavailable. + /// + Enabled = 1, + + /// + /// Never ask; check plain availability only. + /// + Disabled = 2, +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs index f4bf36872..a426fdb78 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -89,6 +89,7 @@ private enum MemberFlags ExplicitOverrideFlag = 1 << 2, // manual failover target (see ExplicitOverride) SkipInitialHealthCheck = 1 << 3, // see SkipInitialHealthCheck Unhealthy = 1 << 4, // disabled by a health-check or circuit-breaker + FailbackVerified = 1 << 5, // most recent failback health-check passed (see FailbackHealthCheck) } private int _flags; @@ -124,6 +125,8 @@ internal void SetUnhealthy() // UTC ticks (DateTime.Ticks and TimeSpan.Ticks share the same 100ns unit); long for atomicity private long _lastUnhealthyTicks; + internal long LastUnhealthyTicks => Volatile.Read(ref _lastUnhealthyTicks); + /// /// Clear the flag against this endpoint, allowing it to be reconsidered. /// @@ -228,6 +231,21 @@ public bool SkipInitialHealthCheck /// public TimeSpan? FailbackDelay { get; set; } + /// + /// A health-check consulted only before traffic moves to this member; when + /// , is used, and when that + /// is also , there is none. + /// + /// + /// Unlike , this never removes the active member: it decides whether a member + /// that is not active may become so, typically by asking whether it has caught up with the others. + /// A member is only eligible once this reports ; an + /// answer (for example, the question could not be asked) + /// leaves it ineligible. If no connected member is eligible, selection falls back to connectivity and + /// alone, so the group always serves from somewhere when it can. + /// + public HealthCheck? FailbackHealthCheck { get; set; } + // the breaker to hand to this member's connections: member override, else its own config, else the group internal CircuitBreaker? ResolveCircuitBreaker(MultiGroupOptions options) => CircuitBreaker ?? Configuration.CircuitBreaker ?? options.CircuitBreaker; @@ -236,6 +254,15 @@ public bool SkipInitialHealthCheck internal TimeSpan ResolveFailbackDelay(MultiGroupOptions options) => FailbackDelay ?? options.FailbackDelay; + internal HealthCheck? ResolveFailbackHealthCheck(MultiGroupOptions options) => FailbackHealthCheck ?? options.FailbackHealthCheck; + + // maintained by the group on each pass; only meaningful when a failback health-check resolves + internal bool FailbackVerified + { + get => GetFlag(MemberFlags.FailbackVerified); + set => SetFlag(MemberFlags.FailbackVerified, value); + } + /// /// The relative weight of this group member; higher is preferred. /// @@ -327,7 +354,10 @@ internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheckResul if (isConnected) { - if (Volatile.Read(ref _lastUnhealthyTicks) < failbackFailureCutoffTicks) ResetIsUnhealthy(); + // at or before the cutoff: with a zero FailbackDelay the cutoff is "now", and on a coarse clock + // (DateTime.UtcNow advances every ~15.6ms on .NET Framework) a failure and the pass that clears + // it can share a tick - a strict comparison would then hold the member out for another interval + if (Volatile.Read(ref _lastUnhealthyTicks) <= failbackFailureCutoffTicks) ResetIsUnhealthy(); } else { @@ -667,8 +697,82 @@ internal async Task RunHealthCheckAsync() } HealthCheck.PutReusablePending(ref _reusableHealthCheckBuffer, ref pending); + await RunFailbackChecksAsync(members).ForAwait(); } + // Failback checks decide eligibility, never health: they run only for members that are up and not + // active (the active member's answer is never used), after liveness has been updated, and a verdict + // is only kept while it can still be acted on - so a member that stops being up, or becomes active, + // must be checked afresh before it is next eligible. Anything short of Healthy, including a + // timeout, leaves the member ineligible; none of it marks a member unhealthy. + private async Task RunFailbackChecksAsync(ConnectionGroupMember[] members) + { + // a provisional active member (chosen by the liveness fallback) is still checked, so that it can + // stop being provisional once it has caught up; a normal active member is not + var active = _activeStub.Active; + var provisional = Volatile.Read(ref _provisional); + List? checks = null; + int totalTimeoutMillis = 0; + foreach (var member in members) + { + var check = member.ResolveFailbackHealthCheck(_options); + if (check is null) continue; + + if (!member.ConsiderActive || (ReferenceEquals(member.Multiplexer, active) && !ReferenceEquals(active, provisional))) + { + member.FailbackVerified = false; + continue; + } + + totalTimeoutMillis = Math.Max(totalTimeoutMillis, check.TotalTimeoutMillis()); + (checks ??= []).Add(new(member, check.CheckHealthAsync(member.Multiplexer))); + } + + if (checks is null) return; + + var all = new Task[checks.Count]; + for (int i = 0; i < all.Length; i++) all[i] = checks[i].Pending; + await Task.WhenAll(all).TimeoutAfter(totalTimeoutMillis).ForAwait(); + + foreach (var item in checks) + { + var member = item.Member; + var pending = item.Pending; + if (pending.IsCompletedSuccessfully) + { + member.FailbackVerified = await pending.ForAwait() is HealthCheckResult.Healthy; + } + else + { + _ = pending.ObserveErrors(); + member.FailbackVerified = false; + } + } + } + + // a named pair rather than a tuple: the library must not reference System.ValueTuple (see SanityCheckTests) + private readonly struct PendingFailbackCheck(ConnectionGroupMember member, Task pending) + { + public ConnectionGroupMember Member { get; } = member; + + public Task Pending { get; } = pending; + } + + // Eligible to become (or stay) active: an explicit failover target always is, as is the active member + // - unless it is provisional, having been chosen by the liveness fallback without ever passing its + // failback check; otherwise a member with a failback check must have passed it on the latest pass. + private bool IsFailbackEligible(ConnectionGroupMember member, ConnectionMultiplexer? active) + => member.ExplicitOverride + || (ReferenceEquals(member.Multiplexer, active) && !ReferenceEquals(active, Volatile.Read(ref _provisional))) + || member.ResolveFailbackHealthCheck(_options) is null + || member.FailbackVerified; + + // The active member when it was chosen by the liveness fallback (no live member was eligible), else null. + // Other clients never select a member that fails its lag check at all; this is the nearest equivalent + // that still serves when every member is behind: such a member is left for an eligible one as soon as + // one is available, and becomes an ordinary active member once it passes its own failback check. + private ConnectionMultiplexer? _provisional; + private long GetFailbackFailureCutoff(ConnectionGroupMember member) { // the minimum last-observed unhealthy time (as UTC ticks) that we'll allow for reconnect; @@ -685,7 +789,7 @@ internal void SelectPreferredGroup() { if (_disposed) return; var existingActive = _activeStub.Active; - ConnectionGroupMember? preferredMember = null, previousMember = null; + ConnectionGroupMember? preferredMember = null, fallbackMember = null, previousMember = null; var members = _members; foreach (var member in members) { @@ -699,10 +803,28 @@ internal void SelectPreferredGroup() member.UpdateLatency(); // this can change passively // (note that when in doubt, we prefer the active muxer, to prevent flapping) - preferredMember = ConnectionGroupMember.Select(preferredMember, member, existingActive); + fallbackMember = ConnectionGroupMember.Select(fallbackMember, member, existingActive); + if (IsFailbackEligible(member, existingActive)) + { + preferredMember = ConnectionGroupMember.Select(preferredMember, member, existingActive); + } } } + // The active member is eligible unless provisional, so this only applies when it is gone, there is + // none, or it is itself provisional: if no live member has passed its failback check - every one + // lagging, say, which is the normal state of a partition between regions - serve from the best live + // member rather than from none, marking it provisional. + if (preferredMember is null) + { + preferredMember = fallbackMember; + Volatile.Write(ref _provisional, fallbackMember?.Multiplexer); + } + else + { + Volatile.Write(ref _provisional, null); + } + SetActive(preferredMember?.Multiplexer); if (preferredMember is not null && !ReferenceEquals(preferredMember, previousMember)) @@ -1331,6 +1453,12 @@ public async Task AddAsync(ConnectionGroupMember member, TextWriter? log = null) { var health = await member.ResolveHealthCheck(_options).CheckHealthAsync(muxer).ConfigureAwait(false); member.UpdateState(health, GetFailbackFailureCutoff(member)); + + // a newly added member is never the active one, so its failback check applies at once + if (member.ResolveFailbackHealthCheck(_options) is { } failback && member.ConsiderActive) + { + member.FailbackVerified = await failback.CheckHealthAsync(muxer).ConfigureAwait(false) is HealthCheckResult.Healthy; + } } // apply any shared hooks diff --git a/src/StackExchange.Redis/Availability/MultiGroupOptions.cs b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs index 2840d996c..0377dd23e 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupOptions.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs @@ -18,7 +18,7 @@ public sealed class MultiGroupOptions internal static readonly TimeSpan DefaultFailbackDelay = TimeSpan.Zero; private static readonly MultiGroupOptions DefaultInstance = new( - HealthCheck.Default, CircuitBreaker.Default, RetryPolicy.Default, DefaultHealthCheckInterval, DefaultFailbackDelay); + HealthCheck.Default, CircuitBreaker.Default, RetryPolicy.Default, DefaultHealthCheckInterval, DefaultFailbackDelay, null); /// /// Default shared options. @@ -30,13 +30,15 @@ private MultiGroupOptions( CircuitBreaker circuitBreaker, RetryPolicy retryPolicy, TimeSpan healthCheckInterval, - TimeSpan failbackDelay) + TimeSpan failbackDelay, + HealthCheck? failbackHealthCheck) { HealthCheck = healthCheck; CircuitBreaker = circuitBreaker; RetryPolicy = retryPolicy; HealthCheckInterval = healthCheckInterval; FailbackDelay = failbackDelay; + FailbackHealthCheck = failbackHealthCheck; } /// @@ -74,6 +76,12 @@ private MultiGroupOptions( /// public TimeSpan FailbackDelay { get; } + /// + /// The failback health check to use for members of the group when no per-member one is specified; + /// (the default) means none. See . + /// + public HealthCheck? FailbackHealthCheck { get; } + /// /// Allows configuration of . /// @@ -96,6 +104,7 @@ public Builder(MultiGroupOptions options) RetryPolicy = options.RetryPolicy; HealthCheckInterval = options.HealthCheckInterval; FailbackDelay = options.FailbackDelay; + FailbackHealthCheck = options.FailbackHealthCheck; } /// @@ -126,6 +135,12 @@ public Builder(MultiGroupOptions options) /// public TimeSpan FailbackDelay { get; set; } = DefaultFailbackDelay; + /// + /// The failback health check to use for members of the group when no per-member one is specified; + /// (the default) means none. See . + /// + public HealthCheck? FailbackHealthCheck { get; set; } + /// /// Create a new options instance. /// @@ -142,12 +157,13 @@ public MultiGroupOptions Create() && ReferenceEquals(CircuitBreaker, CircuitBreaker.Default) && ReferenceEquals(RetryPolicy, RetryPolicy.Default) && HealthCheckInterval == DefaultHealthCheckInterval - && FailbackDelay == DefaultFailbackDelay) + && FailbackDelay == DefaultFailbackDelay + && FailbackHealthCheck is null) { return DefaultInstance; } - return new MultiGroupOptions(HealthCheck, CircuitBreaker, RetryPolicy, HealthCheckInterval, FailbackDelay); + return new MultiGroupOptions(HealthCheck, CircuitBreaker, RetryPolicy, HealthCheckInterval, FailbackDelay, FailbackHealthCheck); } /// diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index d0eae12df..90e12c560 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -561,25 +561,32 @@ public void SetUserPfxCertificate(string userCertificatePath, string? password = #if NET internal static LocalCertificateSelectionCallback CreatePemUserCertificateCallback(string userCertificatePath, string? userKeyPath) + { + var pfx = LoadPemUserCertificate(userCertificatePath, userKeyPath); + return (sender, targetHost, localCertificates, remoteCertificate, acceptableIssuers) => pfx; + } + + internal static X509Certificate2 LoadPemUserCertificate(string userCertificatePath, string? userKeyPath) { // PEM handshakes not universally supported and causes a runtime error about ephemeral certificates; to avoid, export as PFX using var pem = X509Certificate2.CreateFromPemFile(userCertificatePath, userKeyPath); #pragma warning disable SYSLIB0057 // X509 loading - var pfx = new X509Certificate2(pem.Export(X509ContentType.Pfx)); + return new X509Certificate2(pem.Export(X509ContentType.Pfx)); #pragma warning restore SYSLIB0057 // X509 loading - - return (sender, targetHost, localCertificates, remoteCertificate, acceptableIssuers) => pfx; } #endif internal static LocalCertificateSelectionCallback CreatePfxUserCertificateCallback(string userCertificatePath, string? password, X509KeyStorageFlags storageFlags = X509KeyStorageFlags.DefaultKeySet) { -#pragma warning disable SYSLIB0057 // X509 loading - var pfx = new X509Certificate2(userCertificatePath, password ?? "", storageFlags); -#pragma warning restore SYSLIB0057 // X509 loading + var pfx = LoadPfxUserCertificate(userCertificatePath, password, storageFlags); return (sender, targetHost, localCertificates, remoteCertificate, acceptableIssuers) => pfx; } + internal static X509Certificate2 LoadPfxUserCertificate(string userCertificatePath, string? password, X509KeyStorageFlags storageFlags = X509KeyStorageFlags.DefaultKeySet) +#pragma warning disable SYSLIB0057 // X509 loading + => new X509Certificate2(userCertificatePath, password ?? "", storageFlags); +#pragma warning restore SYSLIB0057 // X509 loading + /// /// Create a certificate validation check that checks against the supplied issuer even when not known by the machine. /// @@ -591,7 +598,7 @@ internal static RemoteCertificateValidationCallback TrustIssuerCallback(string i => TrustIssuerCallback(new X509Certificate2(issuerCertificatePath)); #pragma warning restore SYSLIB0057 // X509 loading - private static RemoteCertificateValidationCallback TrustIssuerCallback(X509Certificate2 issuer) + internal static RemoteCertificateValidationCallback TrustIssuerCallback(X509Certificate2 issuer) { if (issuer == null) throw new ArgumentNullException(nameof(issuer)); diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index b8abcea8f..917d71ed6 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -839,4 +839,10 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) EventId = 120, Message = "{Server}: Re-reading topology after {Failures} consecutive connect failures")] internal static partial void LogInformationRefreshingAfterConnectFailures(this ILogger logger, ServerEndPointLogValue server, int failures); + + [LoggerMessage( + Level = LogLevel.Warning, + EventId = 130, + Message = "Lag-aware health check could not ask {RestEndpoint}: {Reason}")] + internal static partial void LogWarningLagAwareInconclusive(this ILogger logger, string restEndpoint, string reason); } diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index ab058de62..8cc5c21b9 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,35 @@ #nullable enable +StackExchange.Redis.Availability.ConnectionGroupMember.FailbackHealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +StackExchange.Redis.Availability.ConnectionGroupMember.FailbackHealthCheck.set -> void +StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.HealthCheckProbeScope.Endpoint = 0 -> StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.HealthCheckProbeScope.Member = 1 -> StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackHealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackHealthCheck.set -> void +StackExchange.Redis.Availability.MultiGroupOptions.FailbackHealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +virtual StackExchange.Redis.Availability.HealthCheckProbe.Scope.get -> StackExchange.Redis.Availability.HealthCheckProbeScope +StackExchange.Redis.Availability.LagAwareOptions +StackExchange.Redis.Availability.LagAwareOptions.CertificateValidation.get -> System.Net.Security.RemoteCertificateValidationCallback? +StackExchange.Redis.Availability.LagAwareOptions.CertificateValidation.set -> void +StackExchange.Redis.Availability.LagAwareOptions.Credentials.get -> System.Func>? +StackExchange.Redis.Availability.LagAwareOptions.Credentials.set -> void +StackExchange.Redis.Availability.LagAwareOptions.DatabaseId.get -> int? +StackExchange.Redis.Availability.LagAwareOptions.DatabaseId.set -> void +StackExchange.Redis.Availability.LagAwareOptions.LagAwareOptions() -> void +StackExchange.Redis.Availability.LagAwareOptions.LagCheck.get -> StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagAwareOptions.LagCheck.set -> void +StackExchange.Redis.Availability.LagAwareOptions.LagTolerance.get -> System.TimeSpan? +StackExchange.Redis.Availability.LagAwareOptions.LagTolerance.set -> void +StackExchange.Redis.Availability.LagAwareOptions.RestEndpoint.get -> System.Uri? +StackExchange.Redis.Availability.LagAwareOptions.RestEndpoint.set -> void +StackExchange.Redis.Availability.LagAwareOptions.TrustIssuer(string! issuerCertificatePath) -> void +StackExchange.Redis.Availability.LagAwareOptions.TrustIssuer(System.Security.Cryptography.X509Certificates.X509Certificate2! issuer) -> void +StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagCheckMode.Auto = 0 -> StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagCheckMode.Disabled = 2 -> StackExchange.Redis.Availability.LagCheckMode +StackExchange.Redis.Availability.LagCheckMode.Enabled = 1 -> StackExchange.Redis.Availability.LagCheckMode +static StackExchange.Redis.Availability.HealthCheck.LagAware(StackExchange.Redis.Availability.LagAwareOptions! options) -> StackExchange.Redis.Availability.HealthCheck! +static StackExchange.Redis.Availability.HealthCheckProbe.LagAware(StackExchange.Redis.Availability.LagAwareOptions! options) -> StackExchange.Redis.Availability.HealthCheckProbe! +StackExchange.Redis.Availability.LagAwareOptions.ClientCertificate.get -> System.Security.Cryptography.X509Certificates.X509Certificate2? +StackExchange.Redis.Availability.LagAwareOptions.ClientCertificate.set -> void +StackExchange.Redis.Availability.LagAwareOptions.SetUserPfxCertificate(string! userCertificatePath, string? password = null) -> void diff --git a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt index ab058de62..04376438f 100644 --- a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +StackExchange.Redis.Availability.LagAwareOptions.SetUserPemCertificate(string! userCertificatePath, string? userKeyPath = null) -> void diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 4a805b866..883ed9b20 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -32,6 +32,10 @@ + + + + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs index 8cddd2841..e07061166 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/ClusterRestClient.cs @@ -1,7 +1,12 @@ using System; using System.Collections.Generic; +using System.IO; using System.Net.Http; using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Net.Security; +using System.Security.Authentication; +using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Text.Json; @@ -13,38 +18,75 @@ namespace StackExchange.Redis.FaultInjector.Tests; /// The cluster's own REST API on port 9443, for the few facts the fault injector does not expose. /// /// -/// Reads only. Anything that *changes* state goes through the injector so it is recorded as a job with an -/// action id - which is also how the console works, and it means a scenario can be reconstructed afterwards -/// from the injector's history rather than from somebody's memory. +/// Reads, plus one deliberate exception: , because the injector has no action for +/// it and it is the narrowest reliable way to make an Active-Active member fall behind. Everything else that +/// *changes* state goes through the injector so it is recorded as a job with an action id - which is also how +/// the console works, and it means a scenario can be reconstructed afterwards from the injector's history rather +/// than from somebody's memory. /// public sealed class ClusterRestClient : IDisposable { private readonly HttpClient _http; - public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials, string? certificateAuthorityPath) + public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials, DirectoryInfo configDirectory) { - var handler = new HttpClientHandler(); - - if (certificateAuthorityPath is not null) + var validate = PinnedCertificateValidation(credentials, configDirectory); + var handler = new HttpClientHandler { - // The cluster's management certificate is self-signed per environment, like the proxy ones. Pin to - // the CA in the config directory rather than accepting anything: this channel carries credentials. - var issuer = X509CertificateLoader.LoadCertificateFromFile(certificateAuthorityPath); - handler.ServerCertificateCustomValidationCallback = (_, certificate, chain, _) => - { - if (certificate is null || chain is null) return false; - chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; - chain.ChainPolicy.CustomTrustStore.Add(issuer); - chain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; - return chain.Build(certificate); - }; - } + ServerCertificateCustomValidationCallback = (request, certificate, chain, errors) => validate(request, certificate, chain, errors), + }; _http = new HttpClient(handler) { BaseAddress = credentials.RestUrl, Timeout = TimeSpan.FromSeconds(30) }; var basic = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{credentials.Username}:{credentials.Password}")); _http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basic); } + /// + /// Validates a cluster's management certificate by pinning it on first use. + /// + /// + /// The cluster's management (cnm) certificate is minted at bootstrap and is self-signed - its issuer is + /// itself, not any CA shipped in the config directory. ca.crt there signs the RESP-plane proxy + /// certificates (a different, coincidentally-named CA), so chain-building the cnm cert against it fails even + /// when it is the right environment. There is nowhere to read the cnm cert's expected fingerprint from ahead + /// of time (env_output.json does not carry it), so this pins on first use and persists the fingerprint next + /// to the rest of the environment's files - the same trust model as an SSH known_hosts entry. A later run + /// against the same environment must present the identical certificate; a changed fingerprint fails loudly + /// rather than silently accepting a new identity for a channel that carries credentials. Keyed by cluster: + /// multi-cluster templates put several clusters behind one directory, and every node of one cluster presents + /// the same certificate. Shared with the lag-aware probe under test, so both trust the same anchor. + /// + public static RemoteCertificateValidationCallback PinnedCertificateValidation( + FaultInjectorEnvironment.ClusterCredentials credentials, DirectoryInfo configDirectory) + { + var pinPath = Path.Combine(configDirectory.FullName, $"cluster-cert.{credentials.ClusterName}.sha256"); + return (_, certificate, _, _) => + { + if (certificate is null) return false; + var actual = Convert.ToHexString(SHA256.HashData(certificate.GetRawCertData())); + + if (File.Exists(pinPath)) + { + var expected = File.ReadAllText(pinPath).Trim(); + if (!string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase)) + { + throw new AuthenticationException( + $"cluster management certificate at {credentials.RestUrl} does not match the pinned " + + $"fingerprint in {pinPath} (expected {expected}, got {actual}). If this environment " + + "was re-provisioned, delete that file and re-run to pin the new certificate."); + } + + return true; + } + + // trust on first use: nothing in the config directory names the expected fingerprint ahead of + // time, so the only available anchor is "whatever this environment presents the first time we + // ask", pinned from here on. + File.WriteAllText(pinPath, actual); + return true; + }; + } + public void Dispose() => _http.Dispose(); /// @@ -69,4 +111,49 @@ public ClusterRestClient(FaultInjectorEnvironment.ClusterCredentials credentials return results; } + /// + /// The uid of the named database on this cluster; uids are not shared across the clusters of an + /// Active-Active database, so always look up per cluster. + /// + public async Task FindDatabaseUidAsync(string name) + { + foreach (var (uid, candidate) in await ListDatabasesAsync()) + { + if (string.Equals(candidate, name, StringComparison.Ordinal)) return uid; + } + + throw new InvalidOperationException($"no database named '{name}' on {_http.BaseAddress}"); + } + + /// + /// Pauses or resumes sync into this cluster's member of an Active-Active database + /// (crdt_sync: paused or enabled), waiting until the database reports it and is active. + /// + /// + /// Measured as narrow and reliably undone: one member, no network change, active again within seconds. + /// Callers must resume in a finally. + /// + public async Task SetCrdtSyncAsync(int uid, string value) + { + using (var put = await _http.PutAsJsonAsync($"/v1/bdbs/{uid}", new Dictionary { ["crdt_sync"] = value })) + { + put.EnsureSuccessStatusCode(); + } + + for (int i = 0; i < 60; i++) + { + using var response = await _http.GetAsync($"/v1/bdbs/{uid}?fields=crdt_sync,status"); + response.EnsureSuccessStatusCode(); + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + if (document.RootElement.GetProperty("crdt_sync").GetString() == value + && document.RootElement.GetProperty("status").GetString() == "active") + { + return; + } + + await Task.Delay(1000); + } + + throw new TimeoutException($"bdb {uid} on {_http.BaseAddress} did not reach crdt_sync={value}"); + } } diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs index 6b1b18b5a..e246ed6cf 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorEnvironment.cs @@ -66,7 +66,13 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) /// advertised endpoint type - which matter because they decide which notification family a database can /// even emit. Tests that provision their own database know what they asked for and need none of this. /// - public ClusterCredentials? Cluster { get; private set; } + public ClusterCredentials? Cluster => Clusters.Count > 0 ? Clusters[0] : null; + + /// + /// Credentials for every cluster in the environment, in env_output.json order (which is the + /// injector's cluster_index); one entry for single-cluster templates, two for Active-Active. + /// + public IReadOnlyList Clusters { get; private set; } = []; private static FaultInjectorEnvironment? Discover() { @@ -94,7 +100,7 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) { CertificateAuthorityPath = FindCertificateAuthority(directory), }; - result.Cluster = ReadClusterCredentials(directory); + result.Clusters = ReadClusterCredentials(directory); return result; } @@ -124,10 +130,10 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) return null; } - private static ClusterCredentials? ReadClusterCredentials(DirectoryInfo directory) + private static IReadOnlyList ReadClusterCredentials(DirectoryInfo directory) { var path = Path.Combine(directory.FullName, "env_output.json"); - if (!File.Exists(path)) return null; + if (!File.Exists(path)) return []; try { @@ -136,26 +142,35 @@ private FaultInjectorEnvironment(DirectoryInfo configDirectory, Uri injectorUrl) // two schemas in the wild: single-cluster templates put outputs at the top level, and the AWS // multi-cluster template nests them under .clusters.value[N]. Both are handled because which one // you get is a property of the template somebody chose, not of anything a test can control. - var root = document.RootElement; - if (root.TryGetProperty("clusters", out var clusters) + var roots = new List(); + if (document.RootElement.TryGetProperty("clusters", out var clusters) && clusters.TryGetProperty("value", out var values) - && values.ValueKind == JsonValueKind.Array - && values.GetArrayLength() > 0) + && values.ValueKind == JsonValueKind.Array) + { + foreach (var value in values.EnumerateArray()) roots.Add(value); + } + else { - root = values[0]; + roots.Add(document.RootElement); } - var name = ReadValue(root, "cluster_name") ?? ReadValue(root, "name"); - var user = ReadValue(root, "username") ?? ReadValue(root, "cluster_username"); - var password = ReadValue(root, "password") ?? ReadValue(root, "cluster_password"); + var result = new List(); + foreach (var root in roots) + { + var name = ReadValue(root, "cluster_name") ?? ReadValue(root, "name"); + var user = ReadValue(root, "username") ?? ReadValue(root, "cluster_username"); + var password = ReadValue(root, "password") ?? ReadValue(root, "cluster_password"); + if (name is null || user is null || password is null) break; // keep indexes aligned with cluster_index + result.Add(new ClusterCredentials(name, user, password)); + } - return name is null || user is null || password is null ? null : new ClusterCredentials(name, user, password); + return result; } catch (Exception) { // a malformed env_output.json is not fatal here: only the tests that need REST enrichment care, // and they report it themselves rather than failing every test in the suite at discovery time - return null; + return []; } } diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs index 8ea143f8e..77e9443f8 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/Environment/FaultInjectorFixture.cs @@ -199,7 +199,7 @@ private async Task SweepOrphansAsync() // actual name is an assumption worth not making when the consequence is deleting the wrong thing. var template = ExistingDatabase.ReadAll(Environment).Values.Select(d => d.BdbId).ToHashSet(); - using var rest = new ClusterRestClient(cluster, Environment.CertificateAuthorityPath); + using var rest = new ClusterRestClient(cluster, Environment.ConfigDirectory); foreach (var (bdbId, name) in await rest.ListDatabasesAsync()) { if (template.Contains(bdbId)) continue; diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/LagAwareFailbackScenarioTests.cs b/tests/StackExchange.Redis.FaultInjector.Tests/LagAwareFailbackScenarioTests.cs new file mode 100644 index 000000000..d14e9daf1 --- /dev/null +++ b/tests/StackExchange.Redis.FaultInjector.Tests/LagAwareFailbackScenarioTests.cs @@ -0,0 +1,131 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.FaultInjector.Tests; + +/// +/// +/// The lag-aware failback check against a real Redis Enterprise Active-Active deployment: a group must not fail +/// back onto a member that is alive but behind, and must fail back once it has caught up. +/// +/// +/// No network fault is injected. The member is made to fall behind by pausing sync into it +/// (crdt_sync: paused through its cluster's REST API) while writes go to the other member - measured as +/// narrow and reliably undone, unlike network_latency, whose cleanup can fail and leave a cluster's DNS +/// and REST API unreachable (see notes/lag-aware/findings.md, section 10). +/// +/// +[Trait("tier", "fault-injector")] +[Trait("scenario", "lag-aware")] +public class LagAwareFailbackScenarioTests(ExistingDatabaseFixture fixture, ITestOutputHelper output) + : IClassFixture +{ + private const string DatabaseName = "re-active-active"; + private static readonly TimeSpan HoldOff = TimeSpan.FromSeconds(15); + + [Fact] + public async Task RefusesToFailBackOntoAStaleMemberAndReturnsOnceCaughtUp() + { + var database = fixture.Require(DatabaseName); + Assert.SkipWhen(database.MemberCount < 2, $"'{database.Key}' lists {database.MemberCount} endpoint(s); this needs two member databases"); + var environment = fixture.Environment; + Assert.SkipWhen(environment.Clusters.Count < 2, $"env_output.json carries credentials for {environment.Clusters.Count} cluster(s); this needs both"); + + // each member's own cluster: the one whose name its database host ends with ("redis-.") + FaultInjectorEnvironment.ClusterCredentials ClusterOf(DatabaseMember member) + => environment.Clusters.Single(c => member.Host.EndsWith("." + c.ClusterName, StringComparison.OrdinalIgnoreCase)); + var clusters = database.Members.Select(ClusterOf).ToArray(); + output.WriteLine($"member0={database.Members[0]} ({clusters[0]}), member1={database.Members[1]} ({clusters[1]})"); + + ConnectionGroupMember Member(int index, double weight) => new( + database.GetClientConfig(environment, MaintenanceNotificationMode.Disabled, index), $"member{index}") + { + Weight = weight, + FailbackHealthCheck = HealthCheck.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential(clusters[index].Username, clusters[index].Password)), + CertificateValidation = ClusterRestClient.PinnedCertificateValidation(clusters[index], environment.ConfigDirectory), + }), + }; + + // member1 is preferred, so "not returning to it" is the observable refusal + var member0 = Member(0, weight: 1); + var member1 = Member(1, weight: 9); + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.FromSeconds(1), + FailbackDelay = TimeSpan.Zero, // only the lag check stands between member1 and a failback + }; + + using var rest = new ClusterRestClient(clusters[1], environment.ConfigDirectory); + var uid = await rest.FindDatabaseUidAsync(DatabaseName); + + using var writer = new TestOutputWriter(output); + await using var group = await ConnectionMultiplexer.ConnectGroupAsync([member0, member1], options, writer); + await WaitForActiveAsync(group, member1, TimeSpan.FromSeconds(30), "the preferred, caught-up member should be selected first"); + + var db = group.GetDatabase(); + RedisKey key = "fi-lag-aware-counter"; + await db.KeyDeleteAsync(key); + + bool paused = false; + try + { + await rest.SetCrdtSyncAsync(uid, "paused"); + paused = true; + output.WriteLine($"{Now()} sync into member1 (bdb {uid} on {clusters[1]}) paused"); + + // leave member1 for member0, then release the override: member1 is the higher weight, alive and + // connected, so the only thing that can keep the group off it is its failback check + Assert.True(group.TryFailoverTo(member0)); + Assert.True(group.TryFailoverTo(null)); + + var watch = Stopwatch.StartNew(); + long written = 0; + while (watch.Elapsed < HoldOff) + { + written = await db.StringIncrementAsync(key); // on member0; member1 cannot see these + Assert.True(member1.IsConnected, "member1 should stay connected: the refusal must be lag, not liveness"); + Assert.True( + group.ActiveMember == member0, + $"{Now()} the group moved to {group.ActiveMember} while member1 was behind (failback verified: {member1.FailbackVerified})"); + await Task.Delay(500, TestContext.Current.CancellationToken); + } + + output.WriteLine($"{Now()} held on member0 for {HoldOff.TotalSeconds:0}s, {written} writes member1 has not seen"); + } + finally + { + if (paused) + { + await rest.SetCrdtSyncAsync(uid, "enabled"); + output.WriteLine($"{Now()} sync into member1 resumed"); + } + } + + await WaitForActiveAsync(group, member1, TimeSpan.FromSeconds(60), "the group should fail back once member1 has caught up"); + var seen = (long)await member1.Multiplexer.GetDatabase().StringGetAsync(key); + output.WriteLine($"{Now()} failed back to member1, which now has the counter at {seen}"); + Assert.True(seen > 0, "member1 should have received member0's writes by the time it was selected"); + await db.KeyDeleteAsync(key); + } + + private async Task WaitForActiveAsync(IConnectionGroup group, ConnectionGroupMember expected, TimeSpan timeout, string because) + { + var watch = Stopwatch.StartNew(); + while (group.ActiveMember != expected && watch.Elapsed < timeout) + { + await Task.Delay(250, TestContext.Current.CancellationToken); + } + + output.WriteLine($"{Now()} active: {group.ActiveMember} after {watch.Elapsed.TotalSeconds:0.0}s (wanted {expected})"); + Assert.True(group.ActiveMember == expected, $"{because}; active is {group.ActiveMember} after {timeout.TotalSeconds:0}s"); + } + + private static string Now() => DateTime.UtcNow.ToString("HH:mm:ss.f"); +} diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md index 0be35486c..95c3d3c05 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/README.md +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -125,6 +125,23 @@ is blackholed is the injector's choice - it evidently picks the first, which is every member's `host:port` and says so in the final assertion message, because a wrong-cluster injection otherwise looks exactly like a failback that never happened. +## Lag-aware failback + +`LagAwareFailbackScenarioTests` checks the lag-aware failback check (`HealthCheck.LagAware`) against the +same `re-active-active` pair: the group must not fail back onto a member that is alive but behind, and must +return once it has caught up. It needs credentials for **both** clusters in `env_output.json` (the AWS +multi-cluster template has them) and skips otherwise. + +No fault is injected. The preferred member is made to fall behind by pausing sync *into* it - `crdt_sync: paused` +through its cluster's REST API, via `ClusterRestClient.SetCrdtSyncAsync` - while writes go to the other member, +and sync is resumed in a `finally`. That is the one state change this tier makes outside the injector, because +the injector has no action for it, and because the obvious alternative is worse: `network_latency` shapes the +node's whole interface and its cleanup can fail, which on one environment left the cluster's DNS and REST API +unreachable until fixed by hand. + +Measured shape of a passing run: on the preferred member first; held off it for the 15 s it was behind, while +alive and connected; failed back about 4 s after sync resumed, by which time it had all of the writes. + ## Three states, deliberately distinct | state | behaviour | diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs index 0c2d4892f..22894e1a7 100644 --- a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -44,6 +44,7 @@ public void ProbeTrafficIsNotCallerFacingButIsRoutedNormally() [InlineData(0, 5, 0, HealthCheckResult.Unhealthy)] // All failures [InlineData(1, 0, 2, HealthCheckResult.Healthy)] // Early success [InlineData(0, 1, 2, HealthCheckResult.Inconclusive)] // Early failure but more probes remain + [InlineData(0, 0, 0, HealthCheckResult.Inconclusive)] // Every probe inconclusive: not a failure public void AnySuccess_EvaluatesCorrectly(int success, int failure, int remaining, HealthCheckResult expected) { var policy = HealthCheckProbePolicy.AnySuccess; @@ -65,6 +66,7 @@ public void AnySuccess_EvaluatesCorrectly(int success, int failure, int remainin [InlineData(1, 0, 2, HealthCheckResult.Inconclusive)] // Success but more probes remain [InlineData(0, 1, 2, HealthCheckResult.Unhealthy)] // Early failure [InlineData(4, 0, 1, HealthCheckResult.Inconclusive)] // Multiple successes but still waiting + [InlineData(0, 0, 0, HealthCheckResult.Inconclusive)] // Every probe inconclusive: not a pass public void AllSuccess_EvaluatesCorrectly(int success, int failure, int remaining, HealthCheckResult expected) { var policy = HealthCheckProbePolicy.AllSuccess; diff --git a/tests/StackExchange.Redis.Tests/LagAwareProbeTests.cs b/tests/StackExchange.Redis.Tests/LagAwareProbeTests.cs new file mode 100644 index 000000000..af95e3135 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/LagAwareProbeTests.cs @@ -0,0 +1,401 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Tests.MultiGroupTests; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The lag-aware probe against a stubbed REST transport. Response shapes are the ones captured from real +/// Redis Enterprise 8.0.22 deployments (see notes/lag-aware/findings.md, section 10), minus credentials. +/// +public class LagAwareProbeTests(ITestOutputHelper log) +{ + private const string Host = "redis-14460.c1.example.com"; + private const int Port = 14460; + + private const string BdbList = """ + [{"endpoints":[{"addr":["54.243.16.166"],"addr_type":"external","dns_name":"redis-14460.c1.example.com","oss_cluster_api_preferred_endpoint_type":"ip","oss_cluster_api_preferred_ip_type":"internal","port":14460,"proxy_policy":"single","uid":"1:1"}],"uid":1}, + {"endpoints":[{"addr":["100.53.190.50"],"addr_type":"external","dns_name":"redis-12706.c1.example.com","oss_cluster_api_preferred_endpoint_type":"ip","oss_cluster_api_preferred_ip_type":"external","port":12706,"proxy_policy":"all-master-shards","uid":"2:1"}],"uid":2}] + """; + + private const string ActiveActive = """{"crdt":true,"uid":1}"""; + private const string NotActiveActive = """{"crdt":false,"uid":1}"""; + private const string Unavailable = """{"description":"BDB 1 is not available","error_code":"bdb_unavailable"}"""; + + // answers by path-and-query; records every request + private sealed class StubHandler(Func> respond) : HttpMessageHandler + { + public StubHandler(Func answer) + : this((request, _) => + { + var (status, body) = answer(request.RequestUri!.PathAndQuery); + var response = new HttpResponseMessage(status); + if (body is not null) response.Content = new StringContent(body, Encoding.UTF8, "application/json"); + return Task.FromResult(response); + }) + { + } + + public ConcurrentQueue Requests { get; } = new(); + + public IEnumerable Paths => Requests.Select(r => r.RequestUri!.PathAndQuery); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Enqueue(request); + return respond(request, cancellationToken); + } + } + + // the cluster as measured: discovery, crdt lookup, then the availability answer + private static Func Cluster(bool activeActive, HttpStatusCode availability, string? availabilityBody = null) + => path => path switch + { + "/v1/bdbs?fields=uid,endpoints" => (HttpStatusCode.OK, BdbList), + "/v1/bdbs/1?fields=uid,crdt" => (HttpStatusCode.OK, activeActive ? ActiveActive : NotActiveActive), + _ when path.StartsWith("/v1/bdbs/1/availability", StringComparison.Ordinal) => (availability, availabilityBody), + _ => (HttpStatusCode.NotFound, null), + }; + + private int _credentialCalls; + + private LagAwareOptions Options(StubHandler handler) => new() + { + Credentials = _ => + { + Interlocked.Increment(ref _credentialCalls); + return new(new NetworkCredential("probe@example.com", "s3cret")); + }, + HttpMessageHandlerFactory = () => handler, + }; + + private async Task<(HealthCheckResult Result, StubHandler Handler)> CheckOnceAsync( + StubHandler handler, LagAwareOptions? options = null, TimeSpan? timeout = null, HealthCheckProbe? probe = null) + { + using var server = new InProcessTestServer(log, endpoint: new DnsEndPoint(Host, Port)); + await using var conn = await server.ConnectAsync(); + probe ??= HealthCheckProbe.LagAware(options ?? Options(handler)); + var context = new HealthCheckContext(conn.GetServer(new DnsEndPoint(Host, Port)), timeout ?? TimeSpan.FromSeconds(3)); + return (await probe.CheckHealthAsync(context), handler); + } + + [Fact] + public void ProbeIsMemberScoped() + => Assert.Equal(HealthCheckProbeScope.Member, HealthCheckProbe.LagAware(Options(new StubHandler(Cluster(true, HttpStatusCode.OK)))).Scope); + + [Fact] + public async Task DiscoversTheDatabaseThenAsksTheLagQuestionOfAnActiveActiveMember() + { + var (result, handler) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK))); + + Assert.Equal(HealthCheckResult.Healthy, result); + Assert.Equal( + ["/v1/bdbs?fields=uid,endpoints", "/v1/bdbs/1?fields=uid,crdt", "/v1/bdbs/1/availability?extend_check=lag"], + handler.Paths); + + // the cluster name, derived from the configured database host; Basic auth on every request + Assert.All(handler.Requests, r => + { + Assert.Equal("https://c1.example.com:9443", r.RequestUri!.GetLeftPart(UriPartial.Authority)); + Assert.Equal("Basic", r.Headers.Authorization?.Scheme); + Assert.Equal(Convert.ToBase64String(Encoding.UTF8.GetBytes("probe@example.com:s3cret")), r.Headers.Authorization?.Parameter); + }); + } + + [Fact] + public async Task NeverIssuesAnUnfilteredDatabaseRead() + { + // measured: unfiltered /v1/bdbs and /v1/crdbs return database passwords, even to db_viewer + var (_, handler) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK))); + Assert.All(handler.Paths, path => + { + Assert.DoesNotContain("/v1/crdbs", path); + if (!path.Contains("/availability")) Assert.Contains("fields=", path); + }); + } + + [Fact] + public async Task ChecksPlainAvailabilityOfADatabaseThatIsNotActiveActive() + { + // measured: the lag check reports every non-Active-Active database unavailable, whatever the tolerance + var (result, handler) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: false, HttpStatusCode.OK))); + Assert.Equal(HealthCheckResult.Healthy, result); + Assert.Equal("/v1/bdbs/1/availability", handler.Paths.Last()); + } + + [Fact] + public async Task DiscoveryIsCachedAcrossPassesAndCredentialsAreReadEachTime() + { + var handler = new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK)); + var probe = HealthCheckProbe.LagAware(Options(handler)); + await CheckOnceAsync(handler, probe: probe); + await CheckOnceAsync(handler, probe: probe); + + Assert.Equal(4, handler.Requests.Count); // three, then just the availability call + Assert.Equal("/v1/bdbs/1/availability?extend_check=lag", handler.Paths.Last()); + Assert.Equal(2, _credentialCalls); // rotation: asked once per check + } + + [Theory] + [InlineData(HttpStatusCode.OK, HealthCheckResult.Healthy)] + [InlineData(HttpStatusCode.ServiceUnavailable, HealthCheckResult.Unhealthy)] // unavailable, or lagging + [InlineData(HttpStatusCode.NotFound, HealthCheckResult.Unhealthy)] + [InlineData(HttpStatusCode.Unauthorized, HealthCheckResult.Inconclusive)] // management plane, not the database + [InlineData(HttpStatusCode.Forbidden, HealthCheckResult.Inconclusive)] + [InlineData(HttpStatusCode.InternalServerError, HealthCheckResult.Inconclusive)] + public async Task MapsTheAvailabilityAnswer(HttpStatusCode status, HealthCheckResult expected) + { + var (result, _) = await CheckOnceAsync(new StubHandler(Cluster(activeActive: true, status, status == HttpStatusCode.ServiceUnavailable ? Unavailable : null))); + Assert.Equal(expected, result); + } + + [Fact] + public async Task NotFoundForgetsTheDiscoveredDatabase() + { + var gone = false; + var handler = new StubHandler(path => gone && path.Contains("/availability") ? (HttpStatusCode.NotFound, null) : Cluster(true, HttpStatusCode.OK)(path)); + var probe = HealthCheckProbe.LagAware(Options(handler)); + Assert.Equal(HealthCheckResult.Healthy, (await CheckOnceAsync(handler, probe: probe)).Result); + + gone = true; + Assert.Equal(HealthCheckResult.Unhealthy, (await CheckOnceAsync(handler, probe: probe)).Result); + + gone = false; + await CheckOnceAsync(handler, probe: probe); + Assert.Equal(2, handler.Paths.Count(p => p == "/v1/bdbs?fields=uid,endpoints")); // rediscovered + } + + [Fact] + public async Task TransportFailureIsInconclusive() + { + var (result, _) = await CheckOnceAsync(new StubHandler((_, _) => throw new HttpRequestException("connection refused"))); + Assert.Equal(HealthCheckResult.Inconclusive, result); + } + + [Fact] + public async Task ASlowAnswerIsInconclusiveWithinTheProbeTimeout() + { + var handler = new StubHandler(async (_, cancellationToken) => + { + await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK); + }); + var watch = System.Diagnostics.Stopwatch.StartNew(); + var (result, _) = await CheckOnceAsync(handler, timeout: TimeSpan.FromMilliseconds(300)); + Assert.Equal(HealthCheckResult.Inconclusive, result); + Assert.True(watch.Elapsed < TimeSpan.FromSeconds(10), $"took {watch.Elapsed}"); + } + + [Fact] + public async Task ADatabaseThatCannotBeFoundIsInconclusive() + { + var (result, handler) = await CheckOnceAsync(new StubHandler(path => (HttpStatusCode.OK, path.Contains("fields=uid,endpoints") ? "[]" : null))); + Assert.Equal(HealthCheckResult.Inconclusive, result); + Assert.Single(handler.Requests); + } + + [Fact] + public async Task ExplicitOptionsSkipDiscoveryAndSendTheTolerance() + { + var handler = new StubHandler(_ => (HttpStatusCode.OK, null)); + var options = Options(handler); + options.RestEndpoint = new Uri("https://cluster.example.com:9443/"); + options.DatabaseId = 7; + options.LagCheck = LagCheckMode.Enabled; + options.LagTolerance = TimeSpan.FromMilliseconds(250); + + var (result, _) = await CheckOnceAsync(handler, options); + Assert.Equal(HealthCheckResult.Healthy, result); + var request = Assert.Single(handler.Requests); + Assert.Equal("https://cluster.example.com:9443/v1/bdbs/7/availability?extend_check=lag&availability_lag_tolerance_ms=250", request.RequestUri!.ToString()); + } + + [Fact] + public async Task DisabledNeverAsksTheLagQuestion() + { + var handler = new StubHandler(Cluster(activeActive: true, HttpStatusCode.OK)); + var options = Options(handler); + options.LagCheck = LagCheckMode.Disabled; + var (_, _) = await CheckOnceAsync(handler, options); + Assert.DoesNotContain(handler.Paths, p => p.Contains("extend_check") || p.Contains("crdt")); + } + + [Theory] + [InlineData("redis-14460.c1.example.com", 14460, "c1.example.com")] // the form Redis Enterprise assigns + [InlineData("REDIS-14460.c1.example.com", 14460, "c1.example.com")] + [InlineData("redis-14460.c1.example.com", 6379, "redis-14460.c1.example.com")] // port does not match + [InlineData("cache.example.com", 14460, "cache.example.com")] // a name of the operator's own + [InlineData("redis-14460.", 14460, "redis-14460.")] + public void DerivesTheClusterNameFromTheDatabaseHost(string host, int port, string expected) + => Assert.Equal(expected, HealthCheckProbe.LagAwareProbe.ClusterHost(host, port)); + + [Fact] + public async Task ARedirectIsReportedNotFollowed() + { + // measured: non-master nodes redirect management routes to the master's internal address + var handler = new StubHandler((request, _) => + { + var response = new HttpResponseMessage(HttpStatusCode.TemporaryRedirect); + response.Headers.Location = new Uri("https://10.0.101.184:9443" + request.RequestUri!.PathAndQuery); + return Task.FromResult(response); + }); + var (result, _) = await CheckOnceAsync(handler); + Assert.Equal(HealthCheckResult.Inconclusive, result); + Assert.Single(handler.Requests); + } + + [Fact] + public void FindsTheDatabaseByAddressAsWellAsByName() + => Assert.Equal(2, HealthCheckProbe.LagAwareProbe.FindUid(BdbList, "100.53.190.50", 12706)); + + // public-only and self-signed (CN=lag-aware-probe-test, private key discarded at generation): enough to show + // the certificate reaching the transport, where generating one is not possible (Mono has no CertificateRequest) + private const string PublicOnlyCertificate = "MIIDITCCAgmgAwIBAgIUbLJxUJKq4SkB9FnIFFLUTKBAaSkwDQYJKoZIhvcNAQELBQAwHzEdMBsGA1UEAwwUbGFnLWF3YXJlLXByb2JlLXRlc3QwIBcNMjYxMDA4MTUzNjU0WhgPMjEyNjA5MTQxNTM2NTRaMB8xHTAbBgNVBAMMFGxhZy1hd2FyZS1wcm9iZS10ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxXfFEzpB82vcUAwfk75Blw1BYG57k9XXIBqa0dx7KxivuF0iSZsSPGViBs6Pwjx7Yvq6+FRZHusFEZ0s3qRbbSf5p7dZhAJ4Oeb5TjqYj7rNQaoCjz2+W4eWbSpVpd8If+g3qy/U+C3gTdIPOUu9biExNj9iN/dUt4o/0S3vgp0iwUiP/E2TcnFRk0SHLgo0Irt1U12FWSpdJA+p65OoG4h/xbsO70qqn/ifyLX8B+eYBOaqQi8myls3MPef2b/pmQlMYUaTGR5kT2EY3pRusTEhdpC1xDrmYxUnxzptgkcAval7HjXEU0sB7c7atGgTaQlUSv8CsrOk/wtQ4Lp54QIDAQABo1MwUTAdBgNVHQ4EFgQUbo8ar3s44TIDl8yYq4Hc8TsTs9UwHwYDVR0jBBgwFoAUbo8ar3s44TIDl8yYq4Hc8TsTs9UwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAVMrCof8H0vBm2UBSxSQreCblVlz41Yinkp1Q2B+1s1CkvTXGhTPS3iJv2FYViJ5RsC36nG5e35vdW3L4r0hjn1lZA32lX2XAYUsrjx7YaQBqvm6qYiK9HqFs2uJUNN7EtDZsWPMN+SRoRAkcQsTZlMvf2qKIkcnulU36wQ+vEdEmLRRsUmIbdPqhT09kJGx0X7jVUmD6PnuAyk8/OJPAGQM+uCvS43Gfdm+ywQ9+rQIDs7F3lU9g7D0o8OS3yY7WY87vMbiwh7eUlDmn74ypCXSTpLTEy4OtP4Th+eXrZGV5+ExmVwzbeWucC5nCMQJ4+ZIpkjW6h0tb/4WTnfSE3w=="; + +#if NET + private static System.Security.Cryptography.X509Certificates.X509Certificate2 CreateClientCertificate() + { + using var rsa = System.Security.Cryptography.RSA.Create(2048); + var request = new System.Security.Cryptography.X509Certificates.CertificateRequest( + "CN=lag-aware-probe-test", rsa, System.Security.Cryptography.HashAlgorithmName.SHA256, System.Security.Cryptography.RSASignaturePadding.Pkcs1); + return request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddHours(1)); + } + + [Fact] + public void LoadsAClientCertificateFromPfx() + { + using var certificate = CreateClientCertificate(); + var path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"lagaware-{Guid.NewGuid():n}.pfx"); + try + { + System.IO.File.WriteAllBytes(path, certificate.Export(System.Security.Cryptography.X509Certificates.X509ContentType.Pfx, "pw")); + var options = new LagAwareOptions(); + options.SetUserPfxCertificate(path, "pw"); + Assert.Equal(certificate.Thumbprint, options.ClientCertificate?.Thumbprint); + Assert.True(options.ClientCertificate!.HasPrivateKey); + } + finally + { + System.IO.File.Delete(path); + } + } + + [Fact] + public void LoadsAClientCertificateFromPem() + { + using var certificate = CreateClientCertificate(); + var dir = System.IO.Directory.CreateTempSubdirectory("lagaware-").FullName; + try + { + var crt = System.IO.Path.Combine(dir, "client.crt"); + var key = System.IO.Path.Combine(dir, "client.key"); + System.IO.File.WriteAllText(crt, certificate.ExportCertificatePem()); + System.IO.File.WriteAllText(key, System.Security.Cryptography.X509Certificates.RSACertificateExtensions.GetRSAPrivateKey(certificate)!.ExportPkcs8PrivateKeyPem()); + var options = new LagAwareOptions(); + options.SetUserPemCertificate(crt, key); + Assert.Equal(certificate.Thumbprint, options.ClientCertificate?.Thumbprint); + Assert.True(options.ClientCertificate!.HasPrivateKey); + } + finally + { + System.IO.Directory.Delete(dir, recursive: true); + } + } +#endif + + [Fact] + public void TheClientCertificateReachesTheTransport() + { +#pragma warning disable SYSLIB0057 // X509 loading; X509CertificateLoader is not available on every test TFM + using var certificate = new System.Security.Cryptography.X509Certificates.X509Certificate2(Convert.FromBase64String(PublicOnlyCertificate)); +#pragma warning restore SYSLIB0057 + using var handler = HealthCheckProbe.LagAwareProbe.CreateHandler(validation: null, certificate); +#if NET + var sockets = Assert.IsType(handler); + Assert.Contains(certificate, sockets.SslOptions.ClientCertificates!.Cast()); + Assert.False(sockets.AllowAutoRedirect); +#else + var client = Assert.IsType(handler); + Assert.Equal(ClientCertificateOption.Manual, client.ClientCertificateOptions); + Assert.Contains(certificate, client.ClientCertificates.Cast()); + Assert.False(client.AllowAutoRedirect); +#endif + } + + [Fact] + public void RejectsInvalidOptions() + { + Assert.Throws(() => HealthCheckProbe.LagAware(null!)); + Assert.Throws(() => HealthCheckProbe.LagAware(new LagAwareOptions())); + Assert.Throws(() => HealthCheckProbe.LagAware(new LagAwareOptions + { + Credentials = _ => new(new NetworkCredential()), + LagTolerance = TimeSpan.FromMilliseconds(-1), + })); + } + + [Fact] + public async Task AsAFailbackCheckItKeepsTrafficOffALaggingMember() + { + // end to end at the stub level: beta has the higher weight, but its cluster reports it lagging + var alphaEp = new DnsEndPoint("redis-14460.c1.example.com", 14460); + var betaEp = new DnsEndPoint("redis-14460.c2.example.com", 14460); + using var alpha = new InProcessTestServer(log, endpoint: alphaEp); + using var beta = new InProcessTestServer(log, endpoint: betaEp); + + var handler = new StubHandler((request, _) => + { + var path = request.RequestUri!.PathAndQuery; + var isC2 = request.RequestUri.Host == "c2.example.com"; // the cluster names, derived from the members' hosts + (HttpStatusCode Status, string? Body) answer = path switch + { + "/v1/bdbs?fields=uid,endpoints" => (HttpStatusCode.OK, isC2 ? BdbList.Replace(".c1.", ".c2.") : BdbList), + "/v1/bdbs/1?fields=uid,crdt" => (HttpStatusCode.OK, ActiveActive), + _ => isC2 ? (HttpStatusCode.ServiceUnavailable, Unavailable) : (HttpStatusCode.OK, null), + }; + var response = new HttpResponseMessage(answer.Status); + if (answer.Body is not null) response.Content = new StringContent(answer.Body); + return Task.FromResult(response); + }); + + // liveness held down until both members are connected, so the first real selection sees the whole group + var gate = new GateProbe(); + MultiGroupOptions groupOptions = new MultiGroupOptions.Builder + { + HealthCheck = new HealthCheck.Builder { Probe = gate, ProbeCount = 1 }, + FailbackHealthCheck = HealthCheck.LagAware(Options(handler)), + HealthCheckInterval = TimeSpan.MaxValue, + }; + ConnectionGroupMember[] members = [new(alpha.GetClientConfig(), "alpha") { Weight = 1 }, new(beta.GetClientConfig(), "beta") { Weight = 9 }]; + await using var group = await ConnectionMultiplexer.ConnectGroupAsync(members, groupOptions); + + var watch = System.Diagnostics.Stopwatch.StartNew(); + while (!Array.TrueForAll(members, m => m.Multiplexer.IsConnected) && watch.Elapsed < TimeSpan.FromSeconds(10)) + { + await Task.Delay(25, TestContext.Current.CancellationToken); + } + gate.Open = true; + await MultiGroupMultiplexer.TryHealthCheckAndSelectPreferredGroupAsync(group); + await GroupWait.AssertConnectedAsync(group); + + Assert.Equal("alpha", group.ActiveMember?.Name); + Assert.False(members[1].FailbackVerified); + Assert.Contains(handler.Requests, r => r.RequestUri!.Host == "c2.example.com" && r.RequestUri.AbsolutePath == "/v1/bdbs/1/availability"); + } + + private sealed class GateProbe : HealthCheckProbe + { + public volatile bool Open; + + public override Task CheckHealthAsync(HealthCheckContext context) => Open ? HealthyTask : UnhealthyTask; + } +} diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/FailbackHealthCheckTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/FailbackHealthCheckTests.cs new file mode 100644 index 000000000..9de4768c5 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/FailbackHealthCheckTests.cs @@ -0,0 +1,333 @@ +using System; +using System.Collections.Concurrent; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; +using static StackExchange.Redis.Server.RedisServer; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +/// +/// The failback health check: consulted only before traffic moves to a member, never evicting the active one, +/// with selection falling back to liveness when no live member is eligible. +/// +public class FailbackHealthCheckTests(ITestOutputHelper log) +{ + private static readonly DnsEndPoint Alpha = new("alpha", 6379), Beta = new("beta", 6379), Gamma = new("gamma", 6379); + + // answers per endpoint as scripted by the test (default Healthy), counting calls + private sealed class ScriptedProbe(HealthCheckProbeScope scope = HealthCheckProbeScope.Endpoint) : HealthCheckProbe + { + private readonly ConcurrentDictionary _answers = new(); + + public ConcurrentDictionary Calls { get; } = new(); + + public override HealthCheckProbeScope Scope => scope; + + public void Set(EndPoint endpoint, HealthCheckResult result) => _answers[endpoint] = result; + + public override Task CheckHealthAsync(HealthCheckContext context) + { + var endpoint = context.Server.EndPoint; + Calls.AddOrUpdate(endpoint, 1, (_, n) => n + 1); + return Task.FromResult(_answers.TryGetValue(endpoint, out var result) ? result : HealthCheckResult.Healthy); + } + } + + private static HealthCheck Single(HealthCheckProbe probe) => new HealthCheck.Builder { Probe = probe, ProbeCount = 1 }; + + private sealed class Fixture : IAsyncDisposable + { + private readonly InProcessTestServer[] _servers; + private readonly DnsEndPoint[] _endpoints; + + public Fixture(ITestOutputHelper log, params (DnsEndPoint Endpoint, double Weight)[] members) + { + _servers = new InProcessTestServer[members.Length]; + _endpoints = Array.ConvertAll(members, m => m.Endpoint); + Members = new ConnectionGroupMember[members.Length]; + for (int i = 0; i < members.Length; i++) + { + _servers[i] = new InProcessTestServer(log, endpoint: members[i].Endpoint); + Members[i] = new(_servers[i].GetClientConfig(), members[i].Endpoint.Host) { Weight = members[i].Weight }; + } + } + + public ConnectionGroupMember[] Members { get; } + + public ScriptedProbe Liveness { get; } = new(); + + public ScriptedProbe Failback { get; } = new(); + + public IConnectionGroup Group { get; private set; } = null!; + + public async Task ConnectAsync() + { + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheck = Single(Liveness), + FailbackHealthCheck = Single(Failback), + HealthCheckInterval = TimeSpan.MaxValue, // passes are driven by the test + }; + // Hold every member down until all of their connections are up, so the first real selection + // sees the whole group: otherwise whichever member connects first can be selected by the liveness + // fallback (provisionally) before the others are up - a race, not the rule under test. + foreach (var endpoint in _endpoints) Liveness.Set(endpoint, HealthCheckResult.Unhealthy); + Group = await ConnectionMultiplexer.ConnectGroupAsync(Members, options); + + var watch = System.Diagnostics.Stopwatch.StartNew(); + while (!Array.TrueForAll(Members, m => m.Multiplexer.IsConnected) && watch.Elapsed < TimeSpan.FromSeconds(10)) + { + await Task.Delay(25, TestContext.Current.CancellationToken); + } + Assert.True(Array.TrueForAll(Members, m => m.Multiplexer.IsConnected), "members did not connect"); + + foreach (var endpoint in _endpoints) Liveness.Set(endpoint, HealthCheckResult.Healthy); + await PassAsync(); + await GroupWait.AssertConnectedAsync(Group); + } + + public void MarkDown(EndPoint endpoint) => Liveness.Set(endpoint, HealthCheckResult.Unhealthy); + + public Task PassAsync() => MultiGroupMultiplexer.TryHealthCheckAndSelectPreferredGroupAsync(Group); + + public string? Active => Group.ActiveMember?.Name; + + public async ValueTask DisposeAsync() + { + if (Group is not null) await Group.DisposeAsync(); + foreach (var server in _servers) server.Dispose(); + } + } + + [Fact] + public async Task IneligibleMemberIsNotSelectedUntilItPasses() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); // beta: higher weight, but behind + await f.ConnectAsync(); + Assert.Equal("alpha", f.Active); + + f.Failback.Set(Beta, HealthCheckResult.Healthy); // caught up + await f.PassAsync(); + Assert.Equal("beta", f.Active); + Assert.False(f.Members[1].IsUnhealthy); // eligibility is not health + } + + [Fact] + public async Task InconclusiveIsNotEligible() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Beta, HealthCheckResult.Inconclusive); // "could not ask" + await f.ConnectAsync(); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task ActiveMemberIsNeverEvictedByItsFailbackCheck() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); + + // checked once, on the first real pass when nothing was active yet; never while active + Assert.True(f.Failback.Calls.TryGetValue(Beta, out var calls)); + Assert.Equal(1, calls); + } + + [Fact] + public async Task NoEligibleMemberFallsBackToLiveness() + { + // every member lagging - the normal state of a partition between regions - must still serve + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + } + + [Fact] + public async Task AFallbackChoiceIsLeftForAnEligibleMember() + { + // other clients never select a member that fails its lag check; the nearest equivalent that still + // serves when every member is behind is to leave a fallback choice as soon as anything is eligible + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); // provisional + + f.Failback.Set(Alpha, HealthCheckResult.Healthy); + await f.PassAsync(); + Assert.Equal("alpha", f.Active); // eligible beats provisional, whatever the weights + + f.Failback.Set(Beta, HealthCheckResult.Healthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); // and ordinary failback by weight resumes + } + + [Fact] + public async Task AProvisionalMemberThatCatchesUpBecomesOrdinary() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); // provisional + + f.Failback.Set(Beta, HealthCheckResult.Healthy); // still checked while provisional + await f.PassAsync(); + Assert.Equal("beta", f.Active); // verified: no longer provisional + + // now an ordinary active member: never evicted by its own failback check + f.Failback.Set(Alpha, HealthCheckResult.Healthy); + f.Failback.Set(Beta, HealthCheckResult.Unhealthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); + } + + [Fact] + public async Task DeadActiveIsReplacedEvenWhenNoSurvivorIsEligible() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + + f.MarkDown(Beta); // the active region is gone... + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); // ...and the survivor reports lag because of it + await f.PassAsync(); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task DeadActivePrefersAnEligibleSurvivor() + { + await using var f = new Fixture(log, (Alpha, 1), (Gamma, 5), (Beta, 9)); + await f.ConnectAsync(); + Assert.Equal("beta", f.Active); + + f.MarkDown(Beta); + f.Failback.Set(Gamma, HealthCheckResult.Unhealthy); // higher weight, but behind + await f.PassAsync(); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task ExplicitFailoverOverridesIneligibility() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + f.Failback.Set(Alpha, HealthCheckResult.Unhealthy); + await f.PassAsync(); + Assert.Equal("beta", f.Active); + + Assert.True(f.Group.TryFailoverTo(f.Members[0])); + Assert.Equal("alpha", f.Active); + } + + [Fact] + public async Task FormerActiveMustBeCheckedAfreshBeforeReturning() + { + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + await f.PassAsync(); // beta active: its verdict is not kept while active + Assert.Equal("beta", f.Active); + + Assert.True(f.Group.TryFailoverTo(f.Members[0])); + Assert.Equal("alpha", f.Active); + + // removing the override reselects at once, but beta has no current verdict, so it is not eligible yet + Assert.True(f.Group.TryFailoverTo(null)); + Assert.Equal("alpha", f.Active); + + await f.PassAsync(); // now checked, and healthy + Assert.Equal("beta", f.Active); + } + + [Fact] + public async Task MemberScopedProbeRunsOncePerMember() + { + using var server = new InProcessTestServer(log) { ServerType = ServerType.Cluster }; + GetHost(server.DefaultEndPoint, out var port); + var other = server.AddEmptyNode(new IPEndPoint(IPAddress.Loopback, port + 1)); + server.Migrate((RedisKey)"scope-key", other); + await using var conn = await server.ConnectAsync(); + Assert.Equal(2, conn.GetServers().Length); + + // an endpoint that is not yet connected is reported unhealthy without being probed, so wait for both + var watch = System.Diagnostics.Stopwatch.StartNew(); + while (!Array.TrueForAll(conn.GetServers(), s => s.IsConnected) && watch.Elapsed < TimeSpan.FromSeconds(10)) + { + await Task.Delay(25, TestContext.Current.CancellationToken); + } + Assert.True(Array.TrueForAll(conn.GetServers(), s => s.IsConnected)); + + var perEndpoint = new ScriptedProbe(HealthCheckProbeScope.Endpoint); + var perMember = new ScriptedProbe(HealthCheckProbeScope.Member); + Assert.Equal(HealthCheckResult.Healthy, await Single(perEndpoint).CheckHealthAsync(conn)); + Assert.Equal(HealthCheckResult.Healthy, await Single(perMember).CheckHealthAsync(conn)); + + Assert.Equal(2, perEndpoint.Calls.Count); + Assert.Single(perMember.Calls); + Assert.All(perMember.Calls.Values, n => Assert.Equal(1, n)); + } + + [Theory] + [InlineData(nameof(HealthCheckProbePolicy.AllSuccess))] + [InlineData(nameof(HealthCheckProbePolicy.AnySuccess))] + [InlineData(nameof(HealthCheckProbePolicy.MajoritySuccess))] + public async Task ARunOfInconclusiveProbesIsInconclusiveUnderEveryPolicy(string policy) + { + // "could not tell" is neither a pass nor a failure, whichever policy collates it + using var server = new InProcessTestServer(log, endpoint: Alpha); + await using var conn = await server.ConnectAsync(); + var probe = new ScriptedProbe(); + probe.Set(Alpha, HealthCheckResult.Inconclusive); + HealthCheck check = new HealthCheck.Builder + { + Probe = probe, + ProbeCount = 3, + ProbeInterval = TimeSpan.Zero, + ProbePolicy = policy switch + { + nameof(HealthCheckProbePolicy.AllSuccess) => HealthCheckProbePolicy.AllSuccess, + nameof(HealthCheckProbePolicy.AnySuccess) => HealthCheckProbePolicy.AnySuccess, + _ => HealthCheckProbePolicy.MajoritySuccess, + }, + }; + + Assert.Equal(HealthCheckResult.Inconclusive, await check.CheckHealthAsync(conn)); + } + + [Fact] + public async Task AFailureInTheSameClockTickAsTheCutoffStillFailsBack() + { + // FailbackDelay zero means immediate: a passing check clears a failure recorded at the cutoff itself, + // which on .NET Framework's ~15.6ms clock is the ordinary case for two passes in quick succession + await using var f = new Fixture(log, (Alpha, 1), (Beta, 9)); + await f.ConnectAsync(); + var member = f.Members[0]; + + member.SetUnhealthy(); + Assert.True(member.IsUnhealthy); + member.UpdateState(HealthCheckResult.Healthy, failbackFailureCutoffTicks: member.LastUnhealthyTicks); + Assert.False(member.IsUnhealthy); + } + + [Fact] + public void OptionsCarryTheFailbackHealthCheck() + { + Assert.Null(MultiGroupOptions.Default.FailbackHealthCheck); + Assert.Same(MultiGroupOptions.Default, new MultiGroupOptions.Builder().Create()); + + var check = Single(new ScriptedProbe()); + MultiGroupOptions options = new MultiGroupOptions.Builder { FailbackHealthCheck = check }; + Assert.Same(check, options.FailbackHealthCheck); + Assert.Same(check, new MultiGroupOptions.Builder(options).Create().FailbackHealthCheck); + } +} From 5625693e9fda944b9ec9060924fc76b1a677d6e6 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 16 Oct 2026 09:08:37 +0100 Subject: [PATCH 05/22] Do not abandon a connection attempt before ConnectTimeout has elapsed (#2894) (#3277) The heartbeat's Connecting branch aborted an in-flight attempt as soon as the retry policy said a retry was due. With the default ExponentialRetry at half the connect timeout that is ~2.5s, so any attempt whose name resolution or TLS handshake took longer than that was killed inside its own ConnectTimeout budget and reported as a connect timeout. A primary DNS server that does not answer costs ~5s per lookup on glibc, so on Linux every reconnect attempt died during resolution and the connection never left Connecting. Require ConnectTimeout to have elapsed before the retry policy may abort an attempt. The socket connect already carries its own ConnectTimeout cancellation; this check bounds TLS and the handshake after it. --- src/StackExchange.Redis/PhysicalBridge.cs | 16 +++++-- .../ConnectionAttemptUnitTests.cs | 47 ++++++++++++++++++- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 2ead01179..7cf39edeb 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -582,11 +582,11 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) private int connectStartTicks; private long connectTimeoutRetryCount = 0; + /// Milliseconds since the current (or most recent) connection attempt began. + private int ConnectElapsedMilliseconds() => unchecked(Environment.TickCount - Volatile.Read(ref connectStartTicks)); + private bool DueForConnectRetry() - { - int connectTimeMilliseconds = unchecked(Environment.TickCount - Volatile.Read(ref connectStartTicks)); - return Multiplexer.RawConfig.ReconnectRetryPolicy.ShouldRetry(Volatile.Read(ref connectTimeoutRetryCount), connectTimeMilliseconds); - } + => Multiplexer.RawConfig.ReconnectRetryPolicy.ShouldRetry(Volatile.Read(ref connectTimeoutRetryCount), ConnectElapsedMilliseconds()); internal void OnHeartbeat(bool ifConnectedOnly) { @@ -618,7 +618,13 @@ internal void OnHeartbeat(bool ifConnectedOnly) switch (state) { case State.Connecting: - if (DueForConnectRetry()) + // The retry policy spaces attempts; it must not cut one short while it is still inside its + // ConnectTimeout budget. The default policy (ExponentialRetry at half the connect timeout) + // would otherwise abandon an attempt after 2.5s - less than name resolution alone takes + // when a DNS server is not answering - and then report it as a connect timeout (#2894). + // The socket connect carries its own ConnectTimeout cancellation; this check is what bounds + // TLS and anything after it. + if (ConnectElapsedMilliseconds() >= Multiplexer.RawConfig.ConnectTimeout && DueForConnectRetry()) { Interlocked.Increment(ref connectTimeoutRetryCount); var ex = ExceptionFactory.UnableToConnect(Multiplexer, "ConnectTimeout", Name); diff --git a/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs b/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs index cb675fb29..cfeb65177 100644 --- a/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ConnectionAttemptUnitTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.ComponentModel; using System.Linq; @@ -118,6 +118,51 @@ public async Task TimedOutAttemptCarriesTheTimeout() } } + [Fact] + public async Task AttemptIsNotAbandonedBeforeConnectTimeout() + { + // never answers the TLS handshake, so every attempt lives until the library gives up on it. The retry policy + // here says "retry after 50ms", but an attempt must still get its full ConnectTimeout: the default policy's + // 2.5s first interval was killing attempts whose name resolution alone took longer than that (#2894) + var listener = new TcpListener(IPAddress.Loopback, 0); // not IDisposable on net481 + listener.Start(); + try + { + const int connectTimeout = 1500; + var options = new ConfigurationOptions + { + EndPoints = { (IPEndPoint)listener.LocalEndpoint }, + Ssl = true, + AbortOnConnectFail = false, + ConnectTimeout = connectTimeout, + HeartbeatInterval = TimeSpan.FromMilliseconds(100), + ReconnectRetryPolicy = new LinearRetry(50), + }; + var attempts = Observe(options); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(options); + + // an abandoned attempt is followed at once by the next, so the gap between consecutive timeouts on one + // connection is how long an attempt was allowed to live + static bool IsInteractiveTimeout(ConnectionAttemptCompletedEventArgs a) + => a.ConnectionType == ConnectionType.Interactive && a.FailureType == ConnectionFailureType.UnableToConnect; + await WaitForAsync(() => attempts.Count(IsInteractiveTimeout) >= 3, timeoutMilliseconds: 20000); + var timedOut = attempts.Where(IsInteractiveTimeout).OrderBy(a => a.SequenceNumber).Take(3).ToArray(); + for (int i = 1; i < timedOut.Length; i++) + { + var lifetime = timedOut[i].CompletedTimeUtc - timedOut[i - 1].CompletedTimeUtc; + output.WriteLine($"attempt lifetime: {lifetime.TotalMilliseconds:0}ms"); + Assert.True( + lifetime >= TimeSpan.FromMilliseconds(connectTimeout - 250), + $"attempt abandoned after {lifetime.TotalMilliseconds:0}ms, inside the {connectTimeout}ms connect timeout"); + } + } + finally + { + listener.Stop(); + } + } + [Fact] public async Task AbandonedAttemptIsNotReportedAsAFailureToConnect() { From 1f13e2d4e1376702c801c0233337e0ecc46f2893 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 16 Oct 2026 10:39:57 +0100 Subject: [PATCH 06/22] Read HELLO "server" before trusting "version" as the Redis API version (#3279) * Read HELLO "server" before trusting "version" as the Redis API version (#3278) Valkey reports its own version as HELLO "version" ("server" => "valkey", "version" => "9.1.2") while INFO still says redis_version:7.2.4; Garnet keeps "version" as the Redis API version ("server" => "redis") and adds "garnet_version". Both the interactive and subscription connections issue HELLO, so the detected version depended on whether the last reply processed was HELLO or INFO, and version-gated commands (DELEX for LockRelease) were sent to servers that don't have them. Now "version" only sets the Redis API version when "server" is redis; for a recognised non-Redis server it feeds the product variant instead, and for an unrecognised server it is ignored. A "_version" HELLO field sets the product variant in the Garnet shape. The in-process test server can emulate both measured shapes, and a HELLO-only test (INFO disabled) pins the rule down on RESP2 and RESP3. * Snapshot the HELLO "version" reader instead of allocating a string Parse it as a Version (no string) once we know "server" is redis; only the vendor path, which stores the string anyway, materializes it. * Parse the HELLO "version" straight from UTF-8 Adds Format.TryParseVersion(ReadOnlySpan): net10.0 has a built-in UTF-8 Version.TryParse; elsewhere a Utf8Parser loop over the dotted components, so no char transcoding and, downlevel, no string. The HELLO handler uses it via the function-pointer TryParseScalar overload; a parity test keeps it in step with the char/string overloads. * Use IsScalar on the snapshot reader instead of a separate flag * Match the HELLO "server" name case-insensitively * Pin the raw Version parts (unspecified = -1) for the UTF-8 parser, and the gate normalisation System.Version treats "7.4" as (7, 4, -1, -1) and "7.4.0" as (7, 4, 0, -1); the UTF-8 overload must produce the same raw parts as the char/string overloads, and IsAtLeast/IsEqual must then treat the unspecified parts as zero. * UTF-8 TryParseVersion: defer to the char overload for exact parity The goal is identical results to the pre-existing char path for identical text, whatever System.Version does with it (-1 for unspecified parts, whitespace and sign tolerance per part), not a tidier parser. net10.0 fast-paths the built-in UTF-8 Version.TryParse; anything it rejects, and every other TFM, transcodes and calls the char overload. On .NET Framework that still allocates a string, which is acceptable. The parity test now covers the whitespace/sign cases. --- .../Enums/ProductVariant.cs | 27 +++++- src/StackExchange.Redis/Format.cs | 45 +++++++++ src/StackExchange.Redis/HelloField.cs | 41 +++++++- src/StackExchange.Redis/LoggerExtensions.cs | 12 +++ src/StackExchange.Redis/ResultProcessor.cs | 81 +++++++++++++++- .../StackExchange.Redis.Tests/FormatTests.cs | 70 ++++++++++++++ .../ProductVariantUnitTests.cs | 96 +++++++++++++++++-- .../ValkeyUnitTests.cs | 8 +- .../StackExchange.Redis.Server/RedisServer.cs | 49 +++++++--- 9 files changed, 402 insertions(+), 27 deletions(-) diff --git a/src/StackExchange.Redis/Enums/ProductVariant.cs b/src/StackExchange.Redis/Enums/ProductVariant.cs index fd3a17086..ca85ee8c3 100644 --- a/src/StackExchange.Redis/Enums/ProductVariant.cs +++ b/src/StackExchange.Redis/Enums/ProductVariant.cs @@ -1,48 +1,71 @@ -// ReSharper disable once CheckNamespace +using System; +using RESPite; + +// ReSharper disable once CheckNamespace namespace StackExchange.Redis; /// /// Indicates the flavor of RESP-server being used, if known. Unknown variants will be reported as . /// Inclusion (or omission) in this list does not imply support for any given variant; nor does it indicate any specific /// relationship with the vendor or rights to use the name. It is provided solely for informational purposes. Identification -/// is not guaranteed, and is based on the server's self-reporting (typically via `INFO`), which may be incomplete or misleading. +/// is not guaranteed, and is based on the server's self-reporting (via `INFO` and `HELLO`), which may be incomplete or misleading. /// public enum ProductVariant { /// /// The original Redis server. This is also the default value if the variant is unknown. /// + [AsciiHash("redis")] Redis, /// /// Valkey is a fork of open-source Redis associated with AWS. /// + [AsciiHash("valkey")] Valkey, /// /// Garnet is a Redis-compatible server from Microsoft. /// + [AsciiHash("garnet")] Garnet, /// /// Dragonfly /// + [AsciiHash("dragonfly")] Dragonfly, /// /// Memurai /// + [AsciiHash("memurai")] Memurai, /// /// Redict /// + [AsciiHash("redict")] Redict, /// /// KeyDB /// + [AsciiHash("keydb")] KeyDB, // if you want to add another variant here, please open an issue with the details (variant name, INFO output, etc.) } + +/// +/// Metadata and parsing methods for . +/// +internal static partial class ProductVariantMetadata +{ + /// + /// Parses the server value reported by HELLO (redis, valkey, etc); this is the vendor's + /// name rather than a protocol token, so the match is case-insensitive. + /// + [AsciiHash(CaseSensitive = false)] + internal static partial bool TryParse(ReadOnlySpan value, out ProductVariant variant); +} diff --git a/src/StackExchange.Redis/Format.cs b/src/StackExchange.Redis/Format.cs index a57bea0cd..8907182df 100644 --- a/src/StackExchange.Redis/Format.cs +++ b/src/StackExchange.Redis/Format.cs @@ -623,6 +623,51 @@ internal static bool TryParseVersion(ReadOnlySpan input, [NotNullWhen(true #endif } + /// + /// Parses a UTF-8 version (7.4.3, say) directly from the wire. This must give exactly the same result as + /// the char overload for the same text (including 's -1 for unspecified parts, and + /// its tolerance of whitespace and signs), because callers used to go via that overload: net10.0 fast-paths + /// the built-in UTF-8 Version.TryParse; everything else transcodes and defers to the char overload + /// (which, on .NET Framework, allocates a string: that's fine). + /// + internal static bool TryParseVersion(ReadOnlySpan input, [NotNullWhen(true)] out Version? version) + { +#if NET10_0_OR_GREATER + if (Version.TryParse(input, out version)) return true; + // anything else (including our major-only extension) defers to the char overload, so the edge cases + // (whitespace, signs, etc) resolve identically +#endif + if (input.IsEmpty) + { + version = null; + return false; + } + char[]? lease = null; + int maxChars = Encoding.UTF8.GetMaxCharCount(input.Length); + Span chars = maxChars <= 64 ? stackalloc char[64] : (lease = ArrayPool.Shared.Rent(maxChars)); + try + { + int len; +#if NET + len = Encoding.UTF8.GetChars(input, chars); +#else + unsafe + { + fixed (byte* bPtr = &MemoryMarshal.GetReference(input)) + fixed (char* cPtr = &MemoryMarshal.GetReference(chars)) + { + len = Encoding.UTF8.GetChars(bPtr, input.Length, cPtr, chars.Length); + } + } +#endif + return TryParseVersion(chars.Slice(0, len), out version); + } + finally + { + if (lease is not null) ArrayPool.Shared.Return(lease); + } + } + internal static bool TryParseVersion(string? input, [NotNullWhen(true)] out Version? version) { if (input is not null) diff --git a/src/StackExchange.Redis/HelloField.cs b/src/StackExchange.Redis/HelloField.cs index e5d730ce4..a7a5c37e8 100644 --- a/src/StackExchange.Redis/HelloField.cs +++ b/src/StackExchange.Redis/HelloField.cs @@ -1,4 +1,4 @@ -using System; +using System; using RESPite; namespace StackExchange.Redis; @@ -15,11 +15,48 @@ internal enum HelloField Unknown = 0, /// - /// Redis server version. + /// The server product: redis for Redis and for servers that present as Redis (Garnet); Valkey reports valkey. + /// + [AsciiHash("server")] + Server, + + /// + /// Server version. This is only the Redis API version when is redis; Valkey reports its own + /// version here (9.1.2, say, while INFO still gives redis_version:7.2.4). /// [AsciiHash("version")] Version, + /// + /// Garnet's own version; Garnet reports server as redis and as the Redis API version. + /// + [AsciiHash("garnet_version")] + GarnetVersion, + + /// + /// Valkey's own version, if reported alongside . + /// + [AsciiHash("valkey_version")] + ValkeyVersion, + + /// + /// Dragonfly's own version, if reported. + /// + [AsciiHash("dragonfly_version")] + DragonflyVersion, + + /// + /// Memurai's own version, if reported. + /// + [AsciiHash("memurai_version")] + MemuraiVersion, + + /// + /// Redict's own version, if reported. + /// + [AsciiHash("redict_version")] + RedictVersion, + /// /// Protocol version (2 or 3). /// diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index 917d71ed6..df661e32e 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -642,6 +642,18 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "{Server}: Auto-configured (HELLO) role: {Role}")] internal static partial void LogInformationAutoConfiguredHelloRole(this ILogger logger, ServerEndPointLogValue server, string role); + [LoggerMessage( + Level = LogLevel.Information, + EventId = 131, + Message = "{Server}: Auto-configured (HELLO) product: {Variant} {Version}")] + internal static partial void LogInformationAutoConfiguredHelloProductVariant(this ILogger logger, ServerEndPointLogValue server, ProductVariant variant, string version); + + [LoggerMessage( + Level = LogLevel.Information, + EventId = 132, + Message = "{Server}: Auto-configured (HELLO) unrecognized server '{ServerName}'; ignoring its version")] + internal static partial void LogInformationAutoConfiguredHelloUnknownServer(this ILogger logger, ServerEndPointLogValue server, string serverName); + // PhysicalBridge logging methods [LoggerMessage( Level = LogLevel.Information, diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 6e8d17fee..cd6992312 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -1152,6 +1152,17 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes // Handle HELLO command (returns array/map of key-value pairs) if (message?.Command == RedisCommand.HELLO && reader.IsAggregate) { + // "version" means different things depending on "server": Redis (and Garnet, which presents as + // "redis" and adds "garnet_version") report the Redis API version, but Valkey reports *its own* + // version ("server" => "valkey", "version" => "9.1.2", while INFO still says redis_version:7.2.4), + // so we can only decide what to do with "version" once the whole map has been read (#3278). + bool serverIsRedis = true; // absent means redis (older servers) + ProductVariant? helloVariant = null, vendorVariant = null; + string? vendorVersion = null; + // a snapshot of the "version" value (IsScalar when present), so we can read it as a Version or a + // string once we know which it is + RespReader helloVersion = default; + var iter = reader.AggregateChildren(); while (iter.MoveNext()) { @@ -1167,9 +1178,40 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes switch (field) { - case HelloField.Version when iter.Value.TryParseScalar(Format.TryParseVersion!, out Version version): - server.Version = version; - Log?.LogInformationAutoConfiguredHelloServerVersion(new(server), version); + case HelloField.Server: + if (iter.Value.TryParseScalar(&ProductVariantMetadata.TryParse, out ProductVariant parsedVariant)) + { + helloVariant = parsedVariant; + serverIsRedis = parsedVariant is ProductVariant.Redis; + } + else + { + serverIsRedis = false; + Log?.LogInformationAutoConfiguredHelloUnknownServer(new(server), iter.Value.ReadString() ?? ""); + } + break; + case HelloField.Version: + helloVersion = iter.Value; + break; + case HelloField.GarnetVersion: + vendorVariant = ProductVariant.Garnet; + vendorVersion = iter.Value.ReadString(); + break; + case HelloField.ValkeyVersion: + vendorVariant = ProductVariant.Valkey; + vendorVersion = iter.Value.ReadString(); + break; + case HelloField.DragonflyVersion: + vendorVariant = ProductVariant.Dragonfly; + vendorVersion = iter.Value.ReadString(); + break; + case HelloField.MemuraiVersion: + vendorVariant = ProductVariant.Memurai; + vendorVersion = iter.Value.ReadString(); + break; + case HelloField.RedictVersion: + vendorVariant = ProductVariant.Redict; + vendorVersion = iter.Value.ReadString(); break; case HelloField.Proto when iter.Value.TryReadInt64(out var i64): connection.SetProtocol(i64 >= 3 ? RedisProtocol.Resp3 : RedisProtocol.Resp2); @@ -1204,6 +1246,39 @@ when server.ServerType.SupportsAutoConfigure() } } + if (serverIsRedis) + { + // "version" is the Redis API version; a "{vendor}_version" field, if any, identifies the product + Version? version; + unsafe + { + // (the null test is for the compiler: the function-pointer form loses the NotNullWhen annotation) + if (helloVersion.IsScalar && helloVersion.TryParseScalar(&Format.TryParseVersion, out version) && version is not null) + { + server.Version = version; + Log?.LogInformationAutoConfiguredHelloServerVersion(new(server), version); + } + } + if (vendorVariant is { } fromField && !string.IsNullOrWhiteSpace(vendorVersion)) + { + server.SetProductVariant(fromField, vendorVersion!); + Log?.LogInformationAutoConfiguredHelloProductVariant(new(server), fromField, vendorVersion!); + } + } + else if (helloVariant is { } fromServer) + { + // a known non-Redis server: "version" is the vendor's own version, and the Redis API version + // (if the server reports one at all) comes from INFO's redis_version + var productVersion = vendorVariant == fromServer && !string.IsNullOrWhiteSpace(vendorVersion) + ? vendorVersion! : helloVersion.IsScalar ? helloVersion.ReadString() : null; + if (!string.IsNullOrWhiteSpace(productVersion)) + { + server.SetProductVariant(fromServer, productVersion!); + Log?.LogInformationAutoConfiguredHelloProductVariant(new(server), fromServer, productVersion!); + } + } + + // (an unrecognized server falls through: we can't know what its "version" means, so we leave it alone) SetResult(message, true); return true; } diff --git a/tests/StackExchange.Redis.Tests/FormatTests.cs b/tests/StackExchange.Redis.Tests/FormatTests.cs index 1e41dcecf..886ea794e 100644 --- a/tests/StackExchange.Redis.Tests/FormatTests.cs +++ b/tests/StackExchange.Redis.Tests/FormatTests.cs @@ -8,6 +8,76 @@ namespace StackExchange.Redis.Tests; public class FormatTests(ITestOutputHelper output) : TestBase(output) { + // System.Version is awkward: unspecified parts are -1, so "7.4" is (7, 4, -1, -1) and "7.4.0" is (7, 4, 0, -1), + // and the inbuilt operators order them differently; the UTF-8 overload (what the HELLO handler uses) must + // produce exactly the same raw parts as the char/string overloads, and the version gates must then treat the + // unspecified parts as zero (see VersionExtensions.IsAtLeast). + [Theory] + [InlineData("7", 7, 0, -1, -1)] // major-only is our extension; System.Version rejects it + [InlineData("7.4", 7, 4, -1, -1)] + [InlineData("7.4.0", 7, 4, 0, -1)] + [InlineData("7.4.3", 7, 4, 3, -1)] + [InlineData("7.4.3.0", 7, 4, 3, 0)] + [InlineData("9.1.2", 9, 1, 2, -1)] + [InlineData("7.4.3.1", 7, 4, 3, 1)] + [InlineData("255.255.65535.65535", 255, 255, 65535, 65535)] + [InlineData("0.0", 0, 0, -1, -1)] + [InlineData("007.040", 7, 40, -1, -1)] + // System.Version uses int.TryParse per part, which tolerates whitespace and a sign; we inherit that, not fix it + [InlineData(" 7.4", 7, 4, -1, -1)] + [InlineData("7.4 ", 7, 4, -1, -1)] + [InlineData("7. 4", 7, 4, -1, -1)] + [InlineData("+7.4", 7, 4, -1, -1)] + [InlineData("7.+4", 7, 4, -1, -1)] + [InlineData(" 7 ", 7, 0, -1, -1)] + [InlineData("", null, null, null, null)] + [InlineData(".", null, null, null, null)] + [InlineData("7.", null, null, null, null)] + [InlineData(".7", null, null, null, null)] + [InlineData("7..4", null, null, null, null)] + [InlineData("1.2.3.4.5", null, null, null, null)] + [InlineData("9.1.2-rc1", null, null, null, null)] + [InlineData("-1.2", null, null, null, null)] + [InlineData("a.b", null, null, null, null)] + [InlineData("1.2.3-preview4", null, null, null, null)] + public void TryParseVersionUtf8MatchesChars(string input, int? major, int? minor, int? build, int? revision) + { + bool success = Format.TryParseVersion(Encoding.UTF8.GetBytes(input), out var fromBytes); + Assert.Equal(major is not null, success); + Assert.Equal(success, Format.TryParseVersion(input.AsSpan(), out var fromChars)); + Assert.Equal(success, Format.TryParseVersion(input, out var fromString)); + + if (success) + { + Assert.NotNull(fromBytes); + Assert.Equal((major, minor, build, revision), (fromBytes.Major, fromBytes.Minor, fromBytes.Build, fromBytes.Revision)); + Assert.Equal(fromBytes, fromChars); // Version.Equals compares all four raw parts, -1 included + Assert.Equal(fromBytes, fromString); + } + else + { + Assert.Null(fromBytes); + Assert.Null(fromChars); + Assert.Null(fromString); + } + } + + [Theory] + [InlineData("7.4", "7.4.0", true, true)] // the unspecified parts count as zero for the gates + [InlineData("7.4.0", "7.4", true, true)] + [InlineData("7.4", "7.4.0.0", true, true)] + [InlineData("7.4.3", "7.4", true, false)] + [InlineData("7.4", "7.4.3", false, false)] + [InlineData("8", "7.4.3", true, false)] + [InlineData("7.4.3", "8", false, false)] + public void ParsedVersionGatesIgnoreUnspecifiedParts(string server, string required, bool atLeast, bool equal) + { + Assert.True(Format.TryParseVersion(Encoding.UTF8.GetBytes(server), out var x)); + Assert.True(Format.TryParseVersion(Encoding.UTF8.GetBytes(required), out var y)); + Assert.Equal(atLeast, x.IsAtLeast(y)); + Assert.Equal(equal, x.IsEqual(y)); + } + public static IEnumerable EndpointData() { // note: the 3rd arg is for formatting; null means "expect the original string" diff --git a/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs b/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs index 828df3435..bd4a6e54d 100644 --- a/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ProductVariantUnitTests.cs @@ -1,10 +1,13 @@ -using System.Linq; +using System; +using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading.Tasks; using Xunit; namespace StackExchange.Redis.Tests; +[RunPerProtocol] public class ProductVariantUnitTests(ITestOutputHelper log) { [Theory] @@ -14,19 +17,76 @@ public class ProductVariantUnitTests(ITestOutputHelper log) public async Task DetectProductVariant(ProductVariant variant) { using var serverObj = new ProductServer(variant, log); - using var conn = await serverObj.ConnectAsync(withPubSub: false); + // with pub/sub, both connections issue HELLO, in either order relative to the interactive INFO + using var conn = await serverObj.ConnectAsync(withPubSub: true); var serverApi = conn.GetServer(conn.GetEndPoints().First()); serverApi.Ping(); var reportedProduct = serverApi.GetProductVariant(out var reportedVersion); Assert.Equal(variant, reportedProduct); - log.WriteLine($"Detected {reportedProduct} version: {reportedVersion}"); + log.WriteLine($"Detected {reportedProduct} version: {reportedVersion}; redis API version: {serverApi.Version}"); if (variant == ProductVariant.Redis) { Assert.Equal(serverObj.VersionString, reportedVersion); } else { - Assert.Equal("1.2.3-preview4", reportedVersion); + Assert.Equal(ProductServer.VendorVersion, reportedVersion); + } + // the Redis API version must not be confused with the vendor's own version (#3278) + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + } + + [Theory] + [InlineData(ProductVariant.Redis)] + [InlineData(ProductVariant.Valkey)] + [InlineData(ProductVariant.Garnet)] + public async Task DetectProductVariantFromHelloAlone(ProductVariant variant) + { + // with INFO disabled, HELLO is the only source of version data, so this pins down what we take from it: + // "version" is the Redis API version only when "server" is redis (Redis, Garnet); Valkey repurposes it (#3278) + using var serverObj = new ProductServer(variant, log); + var config = serverObj.GetClientConfig(withPubSub: false); + var commands = serverObj.GetCommands(); + commands.Remove(nameof(RedisCommand.INFO)); + config.CommandMap = CommandMap.Create(commands); + var fallbackVersion = new Version(6, 0, 0); // HELLO exists from 6.0, and the fake server reports 7.0.0 + config.DefaultVersion = fallbackVersion; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var serverApi = conn.GetServer(conn.GetEndPoints().First()); + await serverApi.PingAsync(); + Assert.Equal(variant, serverApi.GetProductVariant(out var reportedVersion)); + log.WriteLine($"Detected {variant} version: {reportedVersion}; redis API version: {serverApi.Version}"); + switch (variant) + { + case ProductVariant.Redis: + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + break; + case ProductVariant.Garnet: + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + Assert.Equal(ProductServer.VendorVersion, reportedVersion); + break; + case ProductVariant.Valkey: + Assert.Equal(fallbackVersion, serverApi.Version); // HELLO told us nothing about the Redis API version + Assert.Equal(ProductServer.VendorVersion, reportedVersion); + break; + } + } + + [Fact] + public async Task ValkeyVersionIsStableAcrossConnections() + { + // #3278: Valkey reports its own version as HELLO "version", so the detected Redis API version used to + // depend on whether the (subscription) HELLO or the (interactive) INFO reply was processed last + using var serverObj = new ProductServer(ProductVariant.Valkey, log); + for (int i = 0; i < 10; i++) + { + await using var conn = await serverObj.ConnectAsync(withPubSub: true); + var serverApi = conn.GetServer(conn.GetEndPoints().First()); + await serverApi.PingAsync(); + Assert.Equal(serverObj.RedisVersion, serverApi.Version); + Assert.Equal(ProductVariant.Valkey, serverApi.GetProductVariant(out var reportedVersion)); + Assert.Equal(ProductServer.VendorVersion, reportedVersion); } } @@ -73,6 +133,8 @@ public async Task MultiDbSupportMatchesProductVariantAndServerType(ProductVarian private sealed class ProductServer : InProcessTestServer { + public const string VendorVersion = "9.1.2"; // must parse as a Version, or the HELLO-version confusion goes unnoticed + private readonly ProductVariant _variant; public ProductServer(ProductVariant variant, ITestOutputHelper log, ServerType serverType = ServerType.Standalone) @@ -90,16 +152,38 @@ protected override void Info(StringBuilder sb, string section, Server.RedisClien switch (_variant) { case ProductVariant.Garnet: - sb.AppendLine("garnet_version:1.2.3-preview4"); + sb.AppendLine("garnet_version:" + VendorVersion) + .AppendLine("server_name:garnet"); break; case ProductVariant.Valkey: - sb.AppendLine("valkey_version:1.2.3-preview4") + sb.AppendLine("valkey_version:" + VendorVersion) .AppendLine("server_name:valkey"); break; } } } + // HELLO shapes as measured against real servers (#3278): + // Garnet 2.2.2: server => redis, version => 7.4.3 (the Redis API version), garnet_version => 2.2.2 + // Valkey 9.1.2: server => valkey, version => 9.1.2 (Valkey's own version; INFO has redis_version:7.2.4) + protected override string HelloServerName => _variant switch + { + ProductVariant.Valkey => "valkey", + _ => "redis", + }; + + protected override string HelloVersionString => _variant switch + { + ProductVariant.Valkey => VendorVersion, + _ => VersionString, + }; + + protected override IReadOnlyList> HelloExtraFields => _variant switch + { + ProductVariant.Garnet => [new("garnet_version", VendorVersion)], + _ => [], + }; + protected override bool SupportMultiDb(out string err) { switch (_variant) diff --git a/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs b/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs index d9a704a04..0ec3fc0c4 100644 --- a/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ValkeyUnitTests.cs @@ -4,6 +4,7 @@ namespace StackExchange.Redis.Tests; +[RunPerProtocol] public class ValkeyUnitTests(ITestOutputHelper log) { [Theory] @@ -16,7 +17,8 @@ public async Task IdentifyValkeyCluster(ServerType type) var serverApi = client.GetServer(server.DefaultEndPoint); Assert.Equal(type, serverApi.ServerType); Assert.Equal(ProductVariant.Valkey, serverApi.GetProductVariant(out var version)); - Assert.Equal("8.1", version); + Assert.Equal("8.1", version); + Assert.Equal(server.RedisVersion, serverApi.Version); // not the valkey version from HELLO (#3278) } private sealed class InProcessValkeyLikeServer(ITestOutputHelper log) : InProcessTestServer(log) @@ -24,6 +26,10 @@ private sealed class InProcessValkeyLikeServer(ITestOutputHelper log) : InProces // see https://github.com/StackExchange/StackExchange.Redis/pull/3050 protected override string ServerModeKey => "server_mode"; + // Valkey reports its own version as HELLO "version" (see #3278); the name match is case-insensitive + protected override string HelloServerName => "Valkey"; + protected override string HelloVersionString => "8.1"; + protected override void Info(StringBuilder sb, string section, Server.RedisClient? client = null) { base.Info(sb, section, client); diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index 7fe8c46a9..beb4aea65 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -625,24 +625,47 @@ protected virtual TypedRedisValue Hello(RedisClient client, in RedisRequest requ client.IsAuthenticated = isAuthed; client.Name = name; - var reply = TypedRedisValue.Rent(14, out var span, RespPrefix.Map); + var extras = HelloExtraFields; + var reply = TypedRedisValue.Rent(14 + (2 * extras.Count), out var span, RespPrefix.Map); span[0] = TypedRedisValue.BulkString("server"); - span[1] = TypedRedisValue.BulkString("redis"); + span[1] = TypedRedisValue.BulkString(HelloServerName); span[2] = TypedRedisValue.BulkString("version"); - span[3] = TypedRedisValue.BulkString(VersionString); - span[4] = TypedRedisValue.BulkString("proto"); - span[5] = TypedRedisValue.Integer(proto32); - span[6] = TypedRedisValue.BulkString("id"); - span[7] = TypedRedisValue.Integer(client.Id); - span[8] = TypedRedisValue.BulkString("mode"); - span[9] = TypedRedisValue.BulkString(ServerModeValue); - span[10] = TypedRedisValue.BulkString("role"); - span[11] = TypedRedisValue.BulkString(GetPrimaryOf(client?.Node) is null ? "master" : "replica"); - span[12] = TypedRedisValue.BulkString("modules"); - span[13] = TypedRedisValue.EmptyArray(RespPrefix.Array); + span[3] = TypedRedisValue.BulkString(HelloVersionString); + int index = 4; + foreach (var pair in extras) + { + span[index++] = TypedRedisValue.BulkString(pair.Key); + span[index++] = TypedRedisValue.BulkString(pair.Value); + } + span[index++] = TypedRedisValue.BulkString("proto"); + span[index++] = TypedRedisValue.Integer(proto32); + span[index++] = TypedRedisValue.BulkString("id"); + span[index++] = TypedRedisValue.Integer(client.Id); + span[index++] = TypedRedisValue.BulkString("mode"); + span[index++] = TypedRedisValue.BulkString(ServerModeValue); + span[index++] = TypedRedisValue.BulkString("role"); + span[index++] = TypedRedisValue.BulkString(GetPrimaryOf(client?.Node) is null ? "master" : "replica"); + span[index++] = TypedRedisValue.BulkString("modules"); + span[index] = TypedRedisValue.EmptyArray(RespPrefix.Array); return reply; } + /// + /// The server reported by HELLO; Redis (and Garnet) report redis, Valkey reports valkey. + /// + protected virtual string HelloServerName => "redis"; + + /// + /// The version reported by HELLO; this is the Redis API version when + /// is redis, but Valkey reports its own version here. + /// + protected virtual string HelloVersionString => VersionString; + + /// + /// Additional fields reported by HELLO (Garnet adds garnet_version, for example). + /// + protected virtual IReadOnlyList> HelloExtraFields => []; + [RedisCommand(-3)] protected virtual TypedRedisValue Sadd(RedisClient client, in RedisRequest request) { From 84765eede5c49419ed251a9cfc491a2d58667fb9 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 16 Oct 2026 10:44:16 +0100 Subject: [PATCH 07/22] Sentinel soft-fail follow-ups: discovery failure as RedisConnectionException, late sentinel init, clear LastException (#3280) Follow-ups to #3187 (AbortOnConnectFail=false for sentinel), prompted by the investigation of #3271: - GetSentinelMasterConnection with a config that has no endpoints of its own and nothing discovered threw AggregateException(InvalidOperationException "No nodes to consider") from the seed connect. It now throws the discovery RedisConnectionException, as it did before #3187. - A sentinel connection whose initial connect soft-failed never ran InitializeSentinel, because only the connect paths called it; such a connection never subscribed to +switch-master and tracked failovers only through the fallback polling. InitializeSentinel is now idempotent and is also reached from a non-first reconfigure that detects a sentinel. - The discovery failure recorded as LastException at connect time stayed there after recovery and was folded into every later "no connection available" message. It is cleared once the managed connection is restored. - Document what abortConnect means for sentinel connections. --- docs/Configuration.md | 5 +++ .../ConnectionMultiplexer.Sentinel.cs | 37 +++++++++++++++---- .../ConnectionMultiplexer.cs | 7 ++++ .../SentinelConfigTests.cs | 21 +++++++++++ .../SentinelTests.cs | 12 ++++++ 5 files changed, 74 insertions(+), 8 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 72bc9b2b9..2519c0e2e 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -36,6 +36,11 @@ pointing to that primary server that will automatically be updated if the primar var conn = ConnectionMultiplexer.Connect("localhost,serviceName=myprimary"); ``` +`abortConnect` applies to sentinel connections the same way it applies to everything else. With `abortConnect=true` (the default), `Connect` +throws if the sentinels cannot be reached or do not know the service. With `abortConnect=false`, `Connect` returns a multiplexer even then: +it is not connected, operations fail with a connection exception that names the problem, and it connects itself to the primary once the +sentinels become reachable and name one. If you want startup to fail when the sentinels are unavailable, keep `abortConnect=true`. + An overview of mapping between the `string` and `ConfigurationOptions` representation is shown below, but you can switch between them trivially: ```csharp diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs index 40d513a3f..a93f8bd95 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs @@ -15,14 +15,21 @@ public partial class ConnectionMultiplexer internal Timer? sentinelPrimaryReconnectTimer; internal readonly Dictionary sentinelConnectionChildren = new(); internal ConnectionMultiplexer? sentinelConnection; + private int _sentinelInitialized; /// /// Initializes the connection as a Sentinel connection and adds the necessary event handlers to track changes to the managed primaries. /// + /// + /// Runs once. It is reached from the connect paths, and also from the reconfigure loop when a connection + /// that soft-failed its initial connect ( is false) later + /// reaches a sentinel: without that, a connection that started while the sentinels were down would never subscribe + /// to +switch-master, and would track failovers only through the fallback polling. + /// /// The to log to, if any. internal void InitializeSentinel(ILogger? log) { - if (!_isSentinel) + if (!_isSentinel || Interlocked.Exchange(ref _sentinelInitialized, 1) != 0) { return; } @@ -253,8 +260,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co } if (connection is not null) try { connection.Dispose(); } catch { } - endpoints = config.EndPoints.Clone(); - connection = ConnectImpl(config, log, endpoints: endpoints); + connection = SeedConnection(failure); break; } @@ -330,11 +336,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co throw failure; } - if (connection is null) - { - endpoints = config.EndPoints.Clone(); - connection = ConnectImpl(config, log, endpoints: endpoints); - } + connection ??= SeedConnection(failure); connection.LastException = failure; connection.Logger?.LogErrorSyncConnectTimeout(failure, failure.Message); @@ -385,6 +387,20 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co } throw; } + + // AbortOnConnectFail=false with no primary discovered: hand back a multiplexer seeded with whatever endpoints + // the caller gave us (the sentinel addresses, via Connect), for SwitchPrimary to rebuild once discovery works. + // A config with no endpoints of its own has nothing to seed with, and that is the discovery failure the caller + // should see - not the "No nodes to consider" that ConnectImpl would throw at an empty endpoint list. + ConnectionMultiplexer SeedConnection(RedisConnectionException discoveryFailure) + { + if (config.EndPoints.Count == 0) + { + throw discoveryFailure; + } + endpoints = config.EndPoints.Clone(); + return ConnectImpl(config, log, endpoints: endpoints); + } } [System.Diagnostics.CodeAnalysis.SuppressMessage("Roslynator", "RCS1075:Avoid empty catch clause that catches System.Exception.", Justification = "We don't care.")] @@ -423,6 +439,11 @@ internal void OnManagedConnectionRestored(object? sender, ConnectionFailedEventA // we are not aware of, so retry SwitchPrimary(e.EndPoint, connection); } + + // A soft-failed connect (AbortOnConnectFail=false) records the discovery failure as LastException, and + // the exception factory folds that into every "no connection available" message. We are connected to a + // sentinel-approved primary again, so stop reporting it. + connection.LastException = null; } catch (Exception) { diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 970561773..ee4105444 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2038,6 +2038,13 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILog else if (standaloneCount == 0 && sentinelCount > 0) { ServerSelectionStrategy.ServerType = ServerType.Sentinel; + if (!first) + { + // A sentinel connection whose initial connect soft-failed (AbortOnConnectFail=false) is + // only now reaching a sentinel; the connect path that would have wired up the sentinel + // handlers has long since returned. (The first connect leaves that to ConnectImpl.) + InitializeSentinel(log); + } } else if (standaloneCount > 0) { diff --git a/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs b/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs index 814f99719..d1e58b37a 100644 --- a/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/SentinelConfigTests.cs @@ -93,6 +93,27 @@ public void Connect_UnreachableSentinel_AbortEnabled_Throws() Assert.Throws(() => ConnectionMultiplexer.Connect(options)); } + [Fact] + public void GetSentinelMasterConnection_NothingToSeedWith_AbortDisabled_ThrowsTheDiscoveryFailure() + { + // the direct API: a long-lived SentinelConnect, then a per-service config that carries only the service name + // (the data endpoints are sentinel's to supply). With nothing discovered and nothing to seed a multiplexer + // with, the caller should see the discovery failure as a RedisConnectionException - not the + // InvalidOperationException("No nodes to consider") that connecting an empty endpoint list produces. + using var sentinel = ConnectionMultiplexer.SentinelConnect(GetUnreachableSentinelOptions(abortOnConnectFail: false)); + var config = new ConfigurationOptions + { + AbortOnConnectFail = false, + ConnectRetry = 0, + ConnectTimeout = 100, + ServiceName = "unreachable-primary", + }; + + var exception = Assert.Throws(() => sentinel.GetSentinelMasterConnection(config)); + Assert.Equal(ConnectionFailureType.UnableToConnect, exception.FailureType); + Assert.Equal("Sentinel: Failed connecting to configured primary for service: unreachable-primary", exception.Message); + } + private static ConfigurationOptions GetUnreachableSentinelOptions(bool abortOnConnectFail) { var options = new ConfigurationOptions diff --git a/tests/StackExchange.Redis.Tests/SentinelTests.cs b/tests/StackExchange.Redis.Tests/SentinelTests.cs index d99e47db7..ffe6d3705 100644 --- a/tests/StackExchange.Redis.Tests/SentinelTests.cs +++ b/tests/StackExchange.Redis.Tests/SentinelTests.cs @@ -565,6 +565,18 @@ await UntilConditionAsync( () => !conn.GetServerSnapshot().ToArray().Any(x => Equals(x.EndPoint, seedEndPoint))); Assert.DoesNotContain(seedEndPoint, conn.GetEndPoints()); + // the sentinel connection behind it soft-failed too, so the connect path never wired up its handlers; it + // must do that itself on reaching a sentinel, or this connection would never hear +switch-master + var sentinelConn = conn.sentinelConnection!; + var switchMaster = RedisChannel.Literal("+switch-master"); + await UntilConditionAsync( + TimeSpan.FromSeconds(10), + () => sentinelConn.GetSubscriber().SubscribedEndpoint(switchMaster) is not null && conn.LastException is null); + Assert.NotNull(sentinelConn.GetSubscriber().SubscribedEndpoint(switchMaster)); + + // and the discovery failure recorded at connect time is no longer reported now that we have a primary + Assert.Null(conn.LastException); + static bool TryPing(IDatabase db) { try From a3f389a858a5e35ab8aa3f0a4afe1fb13b9bed52 Mon Sep 17 00:00:00 2001 From: rebaseandpanic Date: Wed, 21 Oct 2026 18:08:49 +0300 Subject: [PATCH 08/22] Sentinel: re-resolve the primary when it is demoted on a healthy connection (#3281) * fix(sentinel): re-resolve the primary when it is demoted on a healthy connection A sentinel-managed connection notices an in-place demotion of its primary (INFO replication reports role:slave) but only re-elects among the nodes it already knows. When sentinel promoted a node at an address the client has never seen and the +switch-master announcement was missed, there is no primary among them and nothing asks sentinel again: writes fail with "No connection (requires writable - not eligible for replica)" until restart. The sentinel connection's heartbeat now checks, from local state only, whether each managed connection's sentinel-reported primary is connected and reporting itself as a replica (or was never added). If so, a recovery attempt runs off the heartbeat thread, at most one per managed connection: - it asks sentinel first; no answer: retry on a later heartbeat; - sentinel names a node held as a connected replica (sentinel has not failed over yet, or our view of that node's role is stale): re-read that node's role with one INFO replication, rather than reconfiguring every node once per heartbeat for as long as sentinel lags (where INFO cannot be sent, fall through to the switch rather than ask sentinel forever); - otherwise switch, as +switch-master would have. An attempt is skipped while the managed connection is already reconfiguring, or once either multiplexer is disposed; disconnected primaries are left to the existing reconnect timer. The heartbeat scan also forgets disposed children, as the +switch-master handler does. Tests: an in-process sentinel (InProcessSentinelServer) and a standalone SetReplicaOf on the toy server. The demotion-to-unknown-address test fails without the fix (no primary at all) and passes with it; a second test holds sentinel on the demoted node and checks that sentinel is still polled but the connection is not reconfigured on every heartbeat, then follows sentinel once it catches up. Refs #1891, #2487, #1670 * Review follow-ups: make SwitchPrimary claim the reconfigure gate before reshaping the connection - SwitchPrimary now claims the reconfigure gate (activeConfigCause) BEFORE rewriting the endpoint list or retiring servers, via new TryClaimReconfigure/ReleaseReconfigureClaim and a 'claimed' handover parameter on ReconfigureAsync. Previously a reconfigure that started between RecoverDemotedPrimary's pre-checks and ReconfigureAsync's own CAS would refuse the switch's reconfigure AFTER the connection had been reshaped, leaving retired servers and no primary for at least a heartbeat; concurrent switches could also interleave their Clear/TryAdd endpoint rewrites. A lost race now touches nothing and the heartbeat retries. - RecoverDemotedPrimary hands sentinel's already-validated answer to SwitchPrimary (reportedPrimary), so the switch acts on the answer the guards evaluated instead of re-querying and possibly acting on a different one while sentinel flaps - also removing redundant blocking sentinel round-trips from the claimed section. - A successful demotion recovery clears connection.LastException (as the restore handler does) once the reported primary is connected as a primary, so a stale soft-connect discovery failure stops polluting timeout messages after the client is healthy again. - currentSentinelPrimaryEndPoint is now volatile: it is written by SwitchPrimary on worker/handler threads and read by the heartbeat demotion check. - The disposed-child cleanup shared by the +switch-master handler and the heartbeat sweep is extracted into ForgetDisposedChild. - DemotedPrimaryDoesNotReconfigureWhileSentinelStillNamesIt no longer uses wall-clock quiet detection plus a fixed observation window with an exact event count: it waits for a stretch of three sentinel polls with zero configuration changes, re-baselining on stragglers - deterministic on loaded machines, and faster. - docs/Configuration.md documents how a sentinel-managed connection tracks failovers, including demotion-in-place re-resolution. --------- Co-authored-by: mgravell --- docs/Configuration.md | 7 + .../ConnectionMultiplexer.Sentinel.cs | 292 +++++++++++++++--- .../ConnectionMultiplexer.cs | 30 +- .../InProcessSentinelServer.cs | 147 +++++++++ .../SentinelPrimaryDemotionUnitTests.cs | 127 ++++++++ .../StackExchange.Redis.Server/RedisServer.cs | 33 ++ 6 files changed, 582 insertions(+), 54 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/InProcessSentinelServer.cs create mode 100644 tests/StackExchange.Redis.Tests/SentinelPrimaryDemotionUnitTests.cs diff --git a/docs/Configuration.md b/docs/Configuration.md index 2519c0e2e..ac044e4dc 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -41,6 +41,13 @@ throws if the sentinels cannot be reached or do not know the service. With `abor it is not connected, operations fail with a connection exception that names the problem, and it connects itself to the primary once the sentinels become reachable and name one. If you want startup to fail when the sentinels are unavailable, keep `abortConnect=true`. +A sentinel-managed connection tracks failovers through several signals: the `+switch-master` announcement, a reconnect poll whenever the +connection to the primary is lost, and - because the announcement is delivered over pub/sub and can be missed - a periodic check that +notices when the node last named as primary is still connected but now reports itself as a replica (a failover where nothing disconnected, +such as a primary demoted in place). In that last case the client re-asks sentinel for the current primary and follows its answer, even to +a node at an address it has never seen; while sentinel still names the demoted node (sentinel itself has not yet failed over), the client +keeps asking without tearing down or reconfiguring the healthy connections. + An overview of mapping between the `string` and `ConfigurationOptions` representation is shown below, but you can switch between them trivially: ```csharp diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs index a93f8bd95..31f5296ce 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs @@ -11,11 +11,14 @@ namespace StackExchange.Redis; public partial class ConnectionMultiplexer { - internal EndPoint? currentSentinelPrimaryEndPoint; + // volatile: written by SwitchPrimary on worker/handler threads and read by the heartbeat + // (IsSentinelPrimaryDemoted / RecoverDemotedPrimary) - the demotion check must see the latest switch + internal volatile EndPoint? currentSentinelPrimaryEndPoint; internal Timer? sentinelPrimaryReconnectTimer; internal readonly Dictionary sentinelConnectionChildren = new(); internal ConnectionMultiplexer? sentinelConnection; private int _sentinelInitialized; + private int _demotedPrimaryRecoveryActive; /// /// Initializes the connection as a Sentinel connection and adds the necessary event handlers to track changes to the managed primaries. @@ -62,9 +65,7 @@ internal void InitializeSentinel(ILogger? log) // Is the connection still valid? if (child.IsDisposed) { - child.ConnectionFailed -= OnManagedConnectionFailed; - child.ConnectionRestored -= OnManagedConnectionRestored; - sentinelConnectionChildren.Remove(messageParts[0]); + ForgetDisposedChild(messageParts[0], child); } else { @@ -515,6 +516,167 @@ private void ScheduleSentinelPrimaryReconnect(ConnectionMultiplexer connection, } } + /// + /// Unhooks and forgets a disposed managed connection. Must be called under the + /// lock. + /// + private void ForgetDisposedChild(string serviceName, ConnectionMultiplexer child) + { + child.ConnectionFailed -= OnManagedConnectionFailed; + child.ConnectionRestored -= OnManagedConnectionRestored; + sentinelConnectionChildren.Remove(serviceName); + } + + /// + /// Re-resolves the primary through sentinel for every managed connection whose primary has been demoted + /// while its connection stayed up. + /// + /// + /// + /// Every other recovery path needs a signal that this case does not produce: +switch-master can be + /// missed (pub/sub delivery is not guaranteed, and the sentinel-clients guidance says not to rely on it), + /// and the reconnect timer and the restore handler only run once a connection fails. A primary that is + /// demoted in place, with nothing announced and nothing disconnected, is still noticed by the managed + /// connection - its periodic INFO replication sees the role change - but then only re-elected + /// among the nodes it already knows. When sentinel promoted a node at an address it has never seen, + /// there is no primary among them, and nothing asked sentinel again. + /// + /// + /// Runs on this sentinel connection's heartbeat. The check reads only local state, so sentinel is asked + /// only when the managed connection itself reports the demotion, and at most one recovery per managed + /// connection is in flight. A recovery that does not take - sentinel not yet updated, or a reconfigure + /// already running - is simply retried on a later heartbeat, and the check clears itself once the + /// sentinel-reported primary is connected as a primary. + /// + /// + private void CheckManagedPrimariesDemoted() + { + List? demoted = null; + List? disposed = null; + lock (sentinelConnectionChildren) + { + foreach (var pair in sentinelConnectionChildren) + { + if (pair.Value.IsDisposed) + { + (disposed ??= new List()).Add(pair.Key); + } + else if (IsSentinelPrimaryDemoted(pair.Value)) + { + (demoted ??= new List()).Add(pair.Value); + } + } + + if (disposed is not null) + { + foreach (var serviceName in disposed) + { + ForgetDisposedChild(serviceName, sentinelConnectionChildren[serviceName]); + } + } + } + + if (demoted is null) return; + foreach (var child in demoted) + { + if (Interlocked.CompareExchange(ref child._demotedPrimaryRecoveryActive, 1, 0) != 0) continue; + + // asking sentinel and SwitchPrimary both block; keep that off the heartbeat + ThreadPool.QueueUserWorkItem(_ => + { + try + { + RecoverDemotedPrimary(child); + } + catch (Exception ex) + { + // sentinel unreachable, or either connection disposed meanwhile: a later heartbeat looks again + Trace("Demoted primary recovery failed: " + ex.Message, "Sentinel"); + } + finally + { + Volatile.Write(ref child._demotedPrimaryRecoveryActive, 0); + } + }); + } + } + + /// + /// One recovery attempt for a managed connection whose sentinel-reported primary was seen demoted. + /// + /// + /// Sentinel is asked first, because what to do depends on its answer, and a repeat of an unchanged answer + /// must stay cheap - the attempt runs on every heartbeat until the view clears: + /// + /// no answer: nothing to act on; the next heartbeat asks again. + /// a node we hold as a connected replica: either sentinel has not failed over yet, or our + /// view of that node's role is stale. Re-reading its role settles both, where a switch would reconfigure every + /// node to elect a primary that may not exist - once per second for as long as sentinel lags. Where that read + /// cannot be sent (INFO unavailable), fall through to the switch rather than ask sentinel forever. + /// anything else: switch, exactly as a +switch-master announcement would have. + /// Sentinel's answer is handed to so the switch acts on the answer these guards + /// evaluated, rather than asking again and possibly acting on a different one. + /// + /// A reconfigure already running is left alone: claims the reconfigure gate + /// before touching anything, so a lost race leaves no trace and a later heartbeat retries. + /// + private void RecoverDemotedPrimary(ConnectionMultiplexer connection) + { + if (IsDisposed || connection.IsDisposed || connection.RawConfig.ServiceName is not string serviceName) return; + + // cheap early out before the blocking sentinel query; SwitchPrimary's claim is the authoritative check + if (Volatile.Read(ref connection.activeConfigCause) is not null) return; + + if (GetConfiguredPrimaryForService(serviceName) is not EndPoint reported) return; + if (connection.TryResolveServerEndPoint(reported) is { IsConnected: true, IsReplica: true } replica + && replica.CheckInfoReplication()) + { + return; + } + + // the view may have been put right while sentinel was answering + if (!IsSentinelPrimaryDemoted(connection)) return; + + if (SwitchPrimary(connection.currentSentinelPrimaryEndPoint, connection, reportedPrimary: reported) + && connection.TryResolveServerEndPoint(reported) is { IsConnected: true, IsReplica: false }) + { + // As OnManagedConnectionRestored does: we are connected to a sentinel-approved primary again, so a + // soft-failed connect's discovery failure should stop being folded into every timeout message. + connection.LastException = null; + } + } + + /// + /// Whether the primary sentinel last reported for is connected and reporting + /// itself as a replica, or was never added (a switch to it whose reconfigure did not run). A disconnected + /// primary is not this case: that is the reconnect timer's job. + /// + private static bool IsSentinelPrimaryDemoted(ConnectionMultiplexer connection) + { + if (connection.IsDisposed + || Volatile.Read(ref connection.sentinelPrimaryReconnectTimer) is not null + || connection.currentSentinelPrimaryEndPoint is not EndPoint primary) + { + return false; + } + + // a lookup, not GetServerEndPoint: that would create a server for an endpoint the connection does not have + if (connection.TryResolveServerEndPoint(primary) is { } server) + { + return server.IsConnected && server.IsReplica; + } + + // never added: a switch to it that was refused (another reconfigure held the gate) or threw. A missing server + // alone is not enough evidence - the endpoint sentinel gave may simply be spelled differently from the + // server's own (a DNS name against an address) - so only while nothing else is serving as the primary, + // which keeps a healthy connection from switching on every heartbeat. + foreach (var other in connection.GetServerSnapshot()) + { + if (other.ServerType != ServerType.Sentinel && other.IsConnected && !other.IsReplica) return false; + } + return true; + } + internal EndPoint? GetConfiguredPrimaryForService(string serviceName) => _serverSnapshot // same as GetServerSnapshot, but without forcing span .Where(static s => s.ServerType == ServerType.Sentinel) @@ -543,78 +705,108 @@ private void ScheduleSentinelPrimaryReconnect(ConnectionMultiplexer connection, /// The endpoint responsible for the switch. /// The connection that should be switched over to a new primary endpoint. /// The writer to log to, if any. - internal void SwitchPrimary(EndPoint? switchBlame, ConnectionMultiplexer connection, TextWriter? writer = null) + /// The primary sentinel already reported, when the caller has just asked it; + /// otherwise sentinel is queried here. + /// true when a switch ran (the endpoint set and/or roles were reconfigured); false when + /// the view was already correct, or another reconfigure holds the gate - the callers that retry (the reconnect + /// timer, the demotion heartbeat) get a later attempt either way. + internal bool SwitchPrimary(EndPoint? switchBlame, ConnectionMultiplexer connection, TextWriter? writer = null, EndPoint? reportedPrimary = null) { var logger = Logger.With(writer); if (connection.RawConfig.ServiceName is not string serviceName) { logger?.LogInformationServiceNameNotDefined(); - return; + return false; } - // Get new primary - try twice - EndPoint newPrimaryEndPoint = GetConfiguredPrimaryForService(serviceName) + // Get new primary - try twice (unless the caller already holds sentinel's answer) + EndPoint newPrimaryEndPoint = reportedPrimary + ?? GetConfiguredPrimaryForService(serviceName) ?? GetConfiguredPrimaryForService(serviceName) ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, CommandFlags.None, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); connection.currentSentinelPrimaryEndPoint = newPrimaryEndPoint; - // Trigger a blocking reconfigure for the switch, then refresh the sentinel address list. - // reconfigureAll:true forces every already-connected node to re-read its role before the - // primary is re-elected (see IsStalePrimaryView); reconfigureAll:false matches the legacy - // behavior for the "unknown endpoint" rebuild path. - void TriggerReconfigure(bool reconfigureAll) + // Claim the reconfigure gate BEFORE reshaping the connection. ReconfigureAsync refuses to run while + // another reconfigure holds this gate, so rewriting the endpoint list and retiring servers first would + // leave the connection reshaped but unconfigured whenever we lose that race - and two concurrent + // switches could interleave their endpoint rewrites. Claim first and a lost race touches nothing. + string cause = $"Primary switch {serviceName}"; + if (!connection.TryClaimReconfigure(cause)) { - Trace($"Switching primary to {newPrimaryEndPoint}"); - connection.ReconfigureAsync( - first: false, - reconfigureAll: reconfigureAll, - log: logger, - blame: switchBlame, - cause: $"Primary switch {serviceName}", - publishReconfigure: false, - publishReconfigureFlags: CommandFlags.PreferMaster).Wait(); - - UpdateSentinelAddressList(serviceName); + Trace($"Primary switch for {serviceName} skipped; reconfiguration already in progress"); + return false; } - if (!connection.servers.Contains(newPrimaryEndPoint)) + bool switched = false; + try { - // The sentinel-reported primary is an endpoint we have never seen (e.g. a - // recreated node that came back on a brand-new IP). Rebuild the endpoint set - // from scratch around it and reconfigure. - EndPoint[]? replicaEndPoints = GetReplicasForService(serviceName) - ?? GetReplicasForService(serviceName); + // Trigger a blocking reconfigure for the switch, then refresh the sentinel address list. + // reconfigureAll:true forces every already-connected node to re-read its role before the + // primary is re-elected (see IsStalePrimaryView); reconfigureAll:false matches the legacy + // behavior for the "unknown endpoint" rebuild path. + void TriggerReconfigure(bool reconfigureAll) + { + Trace($"Switching primary to {newPrimaryEndPoint}"); + switched = true; // from here the claim is ReconfigureAsync's to release, however it exits + connection.ReconfigureAsync( + first: false, + reconfigureAll: reconfigureAll, + log: logger, + blame: switchBlame, + cause: cause, + publishReconfigure: false, + publishReconfigureFlags: CommandFlags.PreferMaster, + claimed: true).Wait(); + + UpdateSentinelAddressList(serviceName); + } - connection.EndPoints.Clear(); - connection.EndPoints.TryAdd(newPrimaryEndPoint); - if (replicaEndPoints is not null) + if (!connection.servers.Contains(newPrimaryEndPoint)) { - foreach (var replicaEndPoint in replicaEndPoints) + // The sentinel-reported primary is an endpoint we have never seen (e.g. a + // recreated node that came back on a brand-new IP). Rebuild the endpoint set + // from scratch around it and reconfigure. + EndPoint[]? replicaEndPoints = GetReplicasForService(serviceName) + ?? GetReplicasForService(serviceName); + + connection.EndPoints.Clear(); + connection.EndPoints.TryAdd(newPrimaryEndPoint); + if (replicaEndPoints is not null) { - connection.EndPoints.TryAdd(replicaEndPoint); + foreach (var replicaEndPoint in replicaEndPoints) + { + connection.EndPoints.TryAdd(replicaEndPoint); + } } - } - // Only once sentinel gave us the whole picture: a null replica list is a failed query, not a - // deployment with no replicas, and retiring on one would drop healthy nodes over a hiccup. - if (replicaEndPoints is not null) + // Only once sentinel gave us the whole picture: a null replica list is a failed query, not a + // deployment with no replicas, and retiring on one would drop healthy nodes over a hiccup. + if (replicaEndPoints is not null) + { + RetireServersAbsentFromEndPoints(connection, logger); + } + + TriggerReconfigure(reconfigureAll: false); + } + else if (IsStalePrimaryView(connection, newPrimaryEndPoint)) { - RetireServersAbsentFromEndPoints(connection, logger); + // The sentinel-reported primary IS an endpoint we already know about, but our + // cached view of the topology is stale (failover between known nodes: the old + // primary was demoted to a replica in-place and a former replica was promoted). + // A plain reconfigure with reconfigureAll:false would NOT re-read the role of an + // already-connected node, so the primary election would re-pick the stale demoted + // node. Force a full reconfigure so every connected node's role is refreshed before + // the primary is re-elected. + TriggerReconfigure(reconfigureAll: true); } - TriggerReconfigure(reconfigureAll: false); + return switched; } - else if (IsStalePrimaryView(connection, newPrimaryEndPoint)) + finally { - // The sentinel-reported primary IS an endpoint we already know about, but our - // cached view of the topology is stale (failover between known nodes: the old - // primary was demoted to a replica in-place and a former replica was promoted). - // A plain reconfigure with reconfigureAll:false would NOT re-read the role of an - // already-connected node, so the primary election would re-pick the stale demoted - // node. Force a full reconfigure so every connected node's role is refreshed before - // the primary is re-elected. - TriggerReconfigure(reconfigureAll: true); + // not handed over (view already correct, or we threw before the reconfigure): release it ourselves + if (!switched) connection.ReleaseReconfigureClaim(); } } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index ee4105444..a764e574d 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1401,6 +1401,7 @@ internal void OnHeartbeat() Trace("heartbeat"); CheckTopologyRefreshDue(now); + if (_isSentinel) CheckManagedPrimariesDemoted(); var tmp = GetServerSnapshot(); int token = 0; @@ -1760,6 +1761,20 @@ internal static void ActivateServer(ServerEndPoint server, ILogger? log) } } + /// + /// Claims the reconfigure gate without running a reconfigure, so that work that must not interleave with + /// one (such as rewriting the endpoint list) can make its pre-checks and its + /// side effects atomic with respect to competing reconfigures. A successful claim MUST be handed to + /// with + /// claimed: true (which releases it however it exits), or released via + /// - a leaked claim refuses every later reconfigure. + /// + internal bool TryClaimReconfigure(string cause) => + Interlocked.CompareExchange(ref activeConfigCause, cause, null) == null; + + /// Releases a claim taken by that was never handed to a reconfigure. + internal void ReleaseReconfigureClaim() => Interlocked.Exchange(ref activeConfigCause, null); + internal bool ReconfigureIfNeeded(EndPoint? blame, bool fromBroadcast, string cause, bool publishReconfigure = false, CommandFlags flags = CommandFlags.None) { if (fromBroadcast) @@ -1788,16 +1803,23 @@ internal bool ReconfigureIfNeeded(EndPoint? blame, bool fromBroadcast, string ca public Task ReconfigureAsync(string reason) => ReconfigureAsync(first: false, reconfigureAll: false, log: Logger, blame: null, cause: reason); - internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILogger? log, EndPoint? blame, string cause, bool publishReconfigure = false, CommandFlags publishReconfigureFlags = CommandFlags.None) + internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILogger? log, EndPoint? blame, string cause, bool publishReconfigure = false, CommandFlags publishReconfigureFlags = CommandFlags.None, bool claimed = false) { - if (_isDisposed) throw new ObjectDisposedException(ToString()); + if (_isDisposed) + { + // a caller-held claim must not leak: nothing else would ever release it + if (claimed) ReleaseReconfigureClaim(); + throw new ObjectDisposedException(ToString()); + } bool showStats = log is not null; bool ranThisCall = false; try { - // Note that we *always* exchange the reason (first one counts) to prevent duplicate runs - ranThisCall = Interlocked.CompareExchange(ref activeConfigCause, cause, null) == null; + // Note that we *always* exchange the reason (first one counts) to prevent duplicate runs; + // claimed means the caller already owns the gate (TryClaimReconfigure) and we run under - and + // release - that claim + ranThisCall = claimed || Interlocked.CompareExchange(ref activeConfigCause, cause, null) == null; if (!ranThisCall) { diff --git a/tests/StackExchange.Redis.Tests/InProcessSentinelServer.cs b/tests/StackExchange.Redis.Tests/InProcessSentinelServer.cs new file mode 100644 index 000000000..8fecb98e6 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/InProcessSentinelServer.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using RESPite.Messages; +using StackExchange.Redis.Configuration; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// An in-process sentinel that monitors one service on an : it answers the +/// SENTINEL queries a client makes from the monitored server's topology, and does nothing on its own - +/// no failover, no +switch-master. Which node is the primary is whatever the test says it is, which is +/// what lets a test put sentinel and the client's cached view out of step. +/// +public sealed class InProcessSentinelServer : InProcessTestServer +{ + private readonly InProcessTestServer _monitored; + private EndPoint _primary; + + public InProcessSentinelServer(InProcessTestServer monitored, string serviceName, EndPoint primary, EndPoint endpoint, ITestOutputHelper? log = null) + : base(log, endpoint) + { + _monitored = monitored ?? throw new ArgumentNullException(nameof(monitored)); + ServiceName = serviceName ?? throw new ArgumentNullException(nameof(serviceName)); + _primary = primary ?? throw new ArgumentNullException(nameof(primary)); + ServerType = ServerType.Sentinel; + } + + public string ServiceName { get; } + + /// + /// The primary this sentinel reports for ; changing it announces nothing. + /// + public EndPoint Primary + { + get => Volatile.Read(ref _primary); + set => Volatile.Write(ref _primary, value ?? throw new ArgumentNullException(nameof(value))); + } + + /// + /// A tunnel that routes each endpoint to whichever of these servers owns it, so that one client configuration + /// can reach the sentinel and the nodes it monitors. + /// + public static Tunnel CreateTunnel(params InProcessTestServer[] servers) => new RoutingTunnel(servers); + + private int _primaryQueries; + + /// How many times a client has asked this sentinel for the primary. + public int PrimaryQueries => Volatile.Read(ref _primaryQueries); + + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "get-master-addr-by-name", LockFree = true)] + private TypedRedisValue SentinelGetMasterAddrByName(RedisClient client, in RedisRequest request) + { + Interlocked.Increment(ref _primaryQueries); + if (!IsService(request)) return TypedRedisValue.NullArray(RespPrefix.Array); + var host = GetHost(Primary, out var port); + var reply = TypedRedisValue.Rent(2, out var span, RespPrefix.Array); + span[0] = TypedRedisValue.BulkString(host); + span[1] = TypedRedisValue.BulkString(port.ToString()); + return reply; + } + + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "replicas", LockFree = true)] + private TypedRedisValue SentinelReplicas(RedisClient client, in RedisRequest request) => Replicas(request); + + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "slaves", LockFree = true)] + private TypedRedisValue SentinelSlaves(RedisClient client, in RedisRequest request) => Replicas(request); + + // only this sentinel exists + [RedisCommand(3, nameof(RedisCommand.SENTINEL), "sentinels", LockFree = true)] + private TypedRedisValue SentinelSentinels(RedisClient client, in RedisRequest request) + => IsService(request) ? TypedRedisValue.EmptyArray(RespPrefix.Array) : TypedRedisValue.Error("ERR No such master with that name"); + + [RedisCommand(2, nameof(RedisCommand.SENTINEL), "masters", LockFree = true)] + private TypedRedisValue SentinelMasters(RedisClient client, in RedisRequest request) + { + var reply = TypedRedisValue.Rent(1, out var span, RespPrefix.Array); + span[0] = Describe(Primary, "master"); + return reply; + } + + private bool IsService(in RedisRequest request) => string.Equals(request.GetString(2), ServiceName, StringComparison.Ordinal); + + private TypedRedisValue Replicas(in RedisRequest request) + { + if (!IsService(request)) return TypedRedisValue.Error("ERR No such master with that name"); + + // sentinel learns the replicas from the primary, so report whatever replicates it right now + var replicas = new List(); + if (_monitored.TryGetNode(Primary, out var primaryNode)) + { + foreach (var endpoint in _monitored.GetEndPoints()) + { + if (_monitored.TryGetNode(endpoint, out var node) && node.PrimaryId == primaryNode.Id) + { + replicas.Add(endpoint); + } + } + } + + var reply = TypedRedisValue.Rent(replicas.Count, out var span, RespPrefix.Array); + for (int i = 0; i < replicas.Count; i++) + { + span[i] = Describe(replicas[i], "slave"); + } + return reply; + } + + private TypedRedisValue Describe(EndPoint endpoint, string flags) + { + var host = GetHost(endpoint, out var port); + var item = TypedRedisValue.Rent(8, out var span, RespPrefix.Array); + span[0] = TypedRedisValue.BulkString("name"); + span[1] = TypedRedisValue.BulkString(flags == "master" ? ServiceName : $"{host}:{port}"); + span[2] = TypedRedisValue.BulkString("ip"); + span[3] = TypedRedisValue.BulkString(host); + span[4] = TypedRedisValue.BulkString("port"); + span[5] = TypedRedisValue.BulkString(port.ToString()); + span[6] = TypedRedisValue.BulkString("flags"); + span[7] = TypedRedisValue.BulkString(flags); + return item; + } + + private sealed class RoutingTunnel(InProcessTestServer[] servers) : Tunnel + { + private Tunnel? Route(EndPoint endpoint) + { + foreach (var server in servers) + { + if (server.TryGetNode(endpoint, out _)) return server.Tunnel; + } + return null; + } + + public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) + => Route(endpoint) is { } tunnel ? tunnel.GetSocketConnectEndpointAsync(endpoint, cancellationToken) : base.GetSocketConnectEndpointAsync(endpoint, cancellationToken); + + public override ValueTask BeforeAuthenticateAsync(EndPoint endpoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) + => Route(endpoint) is { } tunnel ? tunnel.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken) : base.BeforeAuthenticateAsync(endpoint, connectionType, socket, cancellationToken); + } +} diff --git a/tests/StackExchange.Redis.Tests/SentinelPrimaryDemotionUnitTests.cs b/tests/StackExchange.Redis.Tests/SentinelPrimaryDemotionUnitTests.cs new file mode 100644 index 000000000..2a6a879c0 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SentinelPrimaryDemotionUnitTests.cs @@ -0,0 +1,127 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A sentinel-managed connection whose primary is demoted while every connection stays up. Sentinel's own +/// recovery levers - +switch-master, and killing the demoted node's clients - are exactly what is +/// missing here, so the only signal the client gets is the demoted node reporting itself as a replica. +/// +public class SentinelPrimaryDemotionUnitTests(ITestOutputHelper log) +{ + private const string ServiceName = "mymaster"; + + // documentation-range addresses (RFC 5737): a node that is not in-process can never be reached by accident + private static readonly EndPoint PrimaryA = new IPEndPoint(IPAddress.Parse("192.0.2.1"), 6379); + private static readonly EndPoint ReplicaB = new IPEndPoint(IPAddress.Parse("192.0.2.2"), 6379); + private static readonly EndPoint NewNodeC = new IPEndPoint(IPAddress.Parse("192.0.2.3"), 6379); + private static readonly EndPoint SentinelEndPoint = new IPEndPoint(IPAddress.Parse("192.0.2.10"), 26379); + + [Fact] + public async Task DemotedPrimaryFollowsSentinelToPrimaryAtUnknownAddress() + { + using var data = new InProcessTestServer(log, PrimaryA); + data.AddReplicaNode(ReplicaB, PrimaryA); + using var sentinel = new InProcessSentinelServer(data, ServiceName, PrimaryA, SentinelEndPoint, log); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(CreateConfig(data, sentinel)); + Assert.Equal([PrimaryA], ConnectedPrimaries(conn)); + + // the primary moves to a node the client has never been told about - a pod that came back on a new + // address - and the client misses the announcement; no connection is dropped + FailOverToNewNode(data); + sentinel.Primary = NewNodeC; + + await AssertPrimaryBecomesAsync(conn, NewNodeC); + } + + [Fact] + public async Task DemotedPrimaryDoesNotReconfigureWhileSentinelStillNamesIt() + { + using var data = new InProcessTestServer(log, PrimaryA); + data.AddReplicaNode(ReplicaB, PrimaryA); + using var sentinel = new InProcessSentinelServer(data, ServiceName, PrimaryA, SentinelEndPoint, log); + + await using var conn = await ConnectionMultiplexer.ConnectAsync(CreateConfig(data, sentinel)); + int configurationChanges = 0; + conn.ConfigurationChanged += (_, _) => Interlocked.Increment(ref configurationChanges); + + // sentinel has not caught up: it still names the node that is now a replica + FailOverToNewNode(data); + await UntilAsync(() => ConnectedPrimaries(conn).Length == 0, "the demotion was not noticed"); + + // The property under test: the client keeps asking sentinel, but does not reconfigure every node to + // elect a primary that does not exist. Anchor on the sentinel polls themselves rather than wall-clock + // quiet: require a stretch of several polls with zero configuration changes. A straggler reconfigure + // from the demotion itself re-baselines the stretch; the per-heartbeat reconfigure storm this guards + // against would raise a change on every poll, so no clean stretch would ever complete. + int changesBaseline = Volatile.Read(ref configurationChanges); + int queriesBaseline = sentinel.PrimaryQueries; + await UntilAsync( + () => + { + int changesNow = Volatile.Read(ref configurationChanges); + if (changesNow != changesBaseline) + { + changesBaseline = changesNow; + queriesBaseline = sentinel.PrimaryQueries; + return false; + } + return sentinel.PrimaryQueries >= queriesBaseline + 3; + }, + "no stretch of sentinel polls without a reconfigure"); + log.WriteLine($"clean stretch: {sentinel.PrimaryQueries - queriesBaseline} sentinel polls, configuration changes still {changesBaseline}"); + + // and once sentinel does catch up, the connection follows it + sentinel.Primary = NewNodeC; + await AssertPrimaryBecomesAsync(conn, NewNodeC); + } + + private static ConfigurationOptions CreateConfig(InProcessTestServer data, InProcessSentinelServer sentinel) + { + var config = data.GetClientConfig(); + config.EndPoints.Clear(); + config.EndPoints.Add(SentinelEndPoint); + config.ServiceName = ServiceName; + config.Tunnel = InProcessSentinelServer.CreateTunnel(data, sentinel); + config.ConfigCheckSeconds = 1; // how soon the demoted node's role is re-read + return config; + } + + // C is promoted, and A and B follow it; nothing is announced and no connection drops + private static void FailOverToNewNode(InProcessTestServer data) + { + data.AddEmptyNode(NewNodeC); + data.SetReplicaOf(PrimaryA, NewNodeC); + data.SetReplicaOf(ReplicaB, NewNodeC); + } + + private async Task AssertPrimaryBecomesAsync(IConnectionMultiplexer conn, EndPoint expected) + { + var timer = Stopwatch.StartNew(); + await UntilAsync(() => ConnectedPrimaries(conn).SequenceEqual([expected]), $"primary did not become {Format.ToString(expected)}"); + log.WriteLine($"primary is {Format.ToString(expected)} after {timer.ElapsedMilliseconds}ms"); + + Assert.Equal([expected], ConnectedPrimaries(conn)); + Assert.Equal(expected, conn.GetDatabase().IdentifyEndpoint("demotion-key", CommandFlags.DemandMaster)); + } + + private static async Task UntilAsync(Func condition, string failure) + { + var timer = Stopwatch.StartNew(); + while (!condition()) + { + if (timer.Elapsed > TimeSpan.FromSeconds(20)) Assert.Fail($"{failure} within {timer.Elapsed.TotalSeconds:0}s"); + await Task.Delay(100); + } + } + + private static EndPoint[] ConnectedPrimaries(IConnectionMultiplexer conn) + => conn.GetServers().Where(static s => s.IsConnected && !s.IsReplica).Select(static s => s.EndPoint).ToArray(); +} diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index beb4aea65..a0862ff86 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -149,6 +149,39 @@ public void Failover(EndPoint replica) Log($"failover: {Format.ToString(replica)} promoted, {demoted.Host}:{demoted.Port} demoted"); } + /// + /// Makes replicate - or a primary again, when + /// is null - as REPLICAOF does on a standalone server. Nothing is announced and no connection is dropped; + /// a client finds out from ROLE or INFO replication, or not at all. Both nodes are left serving + /// the whole keyspace, as every standalone node does - including one added by , + /// which otherwise owns no slots and would redirect every key. + /// + public void SetReplicaOf(EndPoint node, EndPoint primary) + { + if (ServerType == ServerType.Cluster) throw new InvalidOperationException($"Server mode is {ServerType}; use {nameof(Failover)}"); + if (!TryGetNode(node ?? throw new ArgumentNullException(nameof(node)), out var target)) throw new KeyNotFoundException($"Node not found: {Format.ToString(node)}"); + Node primaryNode = null; + if (primary is not null) + { + if (!TryGetNode(primary, out primaryNode)) throw new KeyNotFoundException($"Node not found: {Format.ToString(primary)}"); + if (ReferenceEquals(primaryNode, target)) throw new ArgumentException("A node cannot replicate itself", nameof(primary)); + } + + target.UpdateSlots(null); + if (primaryNode is null) + { + target.PrimaryId = null; + target.Flags &= ~NodeFlags.Replica; + } + else + { + primaryNode.UpdateSlots(null); + target.PrimaryId = primaryNode.Id; + target.Flags |= NodeFlags.Replica; + } + Log($"replicaof: {Format.ToString(node)} now replicates {(primary is null ? "no one" : Format.ToString(primary))}"); + } + public bool Migrate(Span key, EndPoint to) => Migrate(ServerSelectionStrategy.GetClusterSlot(key), to); public bool Migrate(in RedisKey key, EndPoint to) => Migrate(GetHashSlot(key), to); From 7870ba9a8de317bcc3e3fd0c2bc5bc4dd4fb4390 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 22 Oct 2026 11:11:22 +0100 Subject: [PATCH 09/22] Route as a cluster only with a slot map; take HELLO mode/role from the interactive connection only (#3285) * Route as a cluster only with a slot map; take HELLO mode/role from the interactive connection (#3284) A proxied deployment can answer HELLO with "mode: cluster" while INFO says redis_mode:standalone and every CLUSTER subcommand is refused (Azure Managed Redis with the Enterprise clustering policy). Under RESP3 pub/sub now has its own connection (#3264), whose handshake is HELLO alone: when that leg alone reconnected, its HELLO retyped the server as a cluster with no INFO to follow, the resulting reconfigure put the multiplexer into cluster mode with no slot map, and cross-slot commands were refused client-side. - Cluster *routing* now requires a slot map: servers that identify as a cluster but give us no map are routed as a single standalone server (new event id 133). The map is never cleared once populated, so an established cluster whose refresh is refused (#3275) keeps routing as one. When a map does arrive, stale redundant-primary flags from an earlier election are cleared. - AnyServer accepts a cluster-typed server while routing as standalone (ServerType.IsRoutableAs), so such a server stays a valid target. - HELLO "mode" and "role" are applied only from the interactive connection; the subscription connection does no discovery and must not overwrite server-level facts. Tests: HelloClusterModeUnitTests reproduces the subscription-only reconnect under both protocols (fails under RESP3 before this change); ClusterCommandsRefusedUnitTests asserts standalone routing and a working cross-slot MGET when CLUSTER is refused. The in-process server gains an overridable HelloModeValue so HELLO and INFO can disagree. * Derive the slot-map check from allocation; reject inverted or out-of-range slot ranges The routing decision in ReconfigureAsync needs only "is there any slot map", not a count. Add ServerSelectionStrategy.IsSlotMapEmpty, derived from whether the map has been allocated: the map is allocated only immediately before a write, writers never store null, and nothing clears an entry, so a non-null map always covers at least one slot. CountCoveredSlots remains for the log line in the cluster branch, where it has always been. That derivation is only sound if nothing can allocate without writing, so UpdateClusterRange now ignores an inverted or out-of-range range instead of allocating and then writing nothing (or indexing out of bounds). SlotRange does not validate ordering and the reply-driven paths build ranges straight from what the server sent. SlotMapUnitTests covers the rejected ranges leaving the map unallocated. --- docs/Configuration.md | 4 +- .../ConnectionMultiplexer.cs | 28 +++++- src/StackExchange.Redis/Enums/ServerType.cs | 11 +++ src/StackExchange.Redis/LoggerExtensions.cs | 6 ++ src/StackExchange.Redis/ResultProcessor.cs | 13 ++- .../ServerSelectionStrategy.cs | 17 ++++ .../ClusterCommandsRefusedUnitTests.cs | 9 ++ .../HelloClusterModeUnitTests.cs | 99 +++++++++++++++++++ .../SlotMapUnitTests.cs | 33 +++++++ .../StackExchange.Redis.Server/RedisServer.cs | 8 +- 10 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs create mode 100644 tests/StackExchange.Redis.Tests/SlotMapUnitTests.cs diff --git a/docs/Configuration.md b/docs/Configuration.md index ac044e4dc..1df31a2ae 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -422,6 +422,8 @@ The library determines whether to use RESP3 by: - Otherwise: RESP3 is attempted if `defaultVersion` (6.0 unless overridden) is v6 or above Note that `HELLO` is issued either way: `HELLO 2` when staying on RESP2. The reply tells us the server version, replication role, mode (standalone/sentinel/cluster) and connection -identifier, none of which we would otherwise know without `INFO` or `CONFIG GET` - both of which are in the `@dangerous` ACL category, and are commonly restricted. `HELLO` itself is +identifier, none of which we would otherwise know without `INFO` or `CONFIG GET` - both of which are in the `@dangerous` ACL category, and are commonly restricted. The role and mode +are only taken from the interactive connection (the subscription connection does no discovery), and a reported mode of `cluster` only leads to cluster *routing* once a slot map has +been obtained from `CLUSTER SLOTS`/`CLUSTER NODES`; a proxy that announces a cluster but refuses those commands is routed as a single standalone server. `HELLO` itself is in the `@connection` category. If you need to prevent it (for example when talking to a proxy that does not understand it), disable it at the command-map level with `$hello=` in the configuration string - or specify a `defaultVersion` below 6.0, since `HELLO` did not exist before then. diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index a764e574d..14b2815b0 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1518,7 +1518,7 @@ public IDatabase GetDatabase(int db = -1, object? asyncState = null) for (int i = 0; i < len; i++) { var server = tmp[(int)(((uint)i + startOffset) % len)]; - if (server != null && server.ServerType == serverType && server.IsSelectable(command, allowDisconnected)) + if (server != null && server.ServerType.IsRoutableAs(serverType) && server.IsSelectable(command, allowDisconnected)) { if (server.IsReplica) { @@ -2046,7 +2046,19 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILog } } - if (clusterCount == 0) + // A server *saying* it is a cluster is not enough to route as one: that needs a slot map, and + // proxied deployments can announce "cluster" while refusing every CLUSTER subcommand (AMR with + // the Enterprise clustering policy, #3284). With no map there is nothing to route by and the + // only thing cluster mode would add is rejecting cross-slot commands client-side, which the + // proxy would have accepted. The map is never cleared once populated, so an established + // cluster whose refresh is refused (#3275) keeps routing as one. + bool noSlotMap = ServerSelectionStrategy.IsSlotMapEmpty; + if (clusterCount > 0 && noSlotMap) + { + log?.LogInformationClusterWithoutSlotMap(clusterCount); + } + + if (clusterCount == 0 || noSlotMap) { // Set the serverSelectionStrategy if (RawConfig.Proxy == Proxy.Twemproxy) @@ -2068,7 +2080,7 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILog InitializeSentinel(log); } } - else if (standaloneCount > 0) + else if (standaloneCount > 0 || clusterCount > 0) { ServerSelectionStrategy.ServerType = ServerType.Standalone; } @@ -2097,8 +2109,14 @@ internal async Task ReconfigureAsync(bool first, bool reconfigureAll, ILog else { ServerSelectionStrategy.ServerType = ServerType.Cluster; - long coveredSlots = ServerSelectionStrategy.CountCoveredSlots(); - log?.LogInformationClusterSlotsCovered(coveredSlots, ServerSelectionStrategy.TotalSlots); + log?.LogInformationClusterSlotsCovered(ServerSelectionStrategy.CountCoveredSlots(), ServerSelectionStrategy.TotalSlots); + + // a cluster has many primaries by design; if we routed as standalone before the map + // arrived, the nomination above may have marked all but one of them redundant + foreach (var primary in primaries) + { + primary.ClearUnselectable(UnselectableFlags.RedundantPrimary); + } } if (!first) { diff --git a/src/StackExchange.Redis/Enums/ServerType.cs b/src/StackExchange.Redis/Enums/ServerType.cs index e7a84ad0a..d7dcccd7b 100644 --- a/src/StackExchange.Redis/Enums/ServerType.cs +++ b/src/StackExchange.Redis/Enums/ServerType.cs @@ -78,5 +78,16 @@ internal static class ServerTypeExtensions ServerType.Envoyproxy => false, _ => true, }; + + /// + /// Whether a server of this type can be sent commands when the multiplexer is routing as . + /// + /// + /// Usually the types match, and the check keeps data traffic away from sentinels; but a server that calls itself + /// a cluster while giving us no slot map is routed as a single standalone server (see #3284), so it must remain a + /// target in that mode. + /// + internal static bool IsRoutableAs(this ServerType server, ServerType routing) + => server == routing || (routing == ServerType.Standalone && server == ServerType.Cluster); } } diff --git a/src/StackExchange.Redis/LoggerExtensions.cs b/src/StackExchange.Redis/LoggerExtensions.cs index df661e32e..8916f25de 100644 --- a/src/StackExchange.Redis/LoggerExtensions.cs +++ b/src/StackExchange.Redis/LoggerExtensions.cs @@ -280,6 +280,12 @@ internal static void LogWithThreadPoolStats(this ILogger? log, string message) Message = "Cluster: {CoveredSlots} of {TotalSlots} slots covered")] internal static partial void LogInformationClusterSlotsCovered(this ILogger logger, long coveredSlots, int totalSlots); + [LoggerMessage( + Level = LogLevel.Information, + EventId = 133, + Message = "{ClusterCount} server(s) identify as a cluster but no slot map is available; routing as standalone")] + internal static partial void LogInformationClusterWithoutSlotMap(this ILogger logger, int clusterCount); + [LoggerMessage( Level = LogLevel.Information, EventId = 35, diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index cd6992312..e985b10f2 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -1157,6 +1157,11 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes // version ("server" => "valkey", "version" => "9.1.2", while INFO still says redis_version:7.2.4), // so we can only decide what to do with "version" once the whole map has been read (#3278). bool serverIsRedis = true; // absent means redis (older servers) + // "mode" and "role" describe the *server*, and only the interactive connection does discovery: + // the subscription connection's handshake is HELLO alone, with no INFO to follow, so a proxy + // whose HELLO disagrees with its INFO (AMR says "cluster" to one and "standalone" to the other, + // #3284) would otherwise have the last word every time pub/sub reconnects on its own + bool helloFromInteractive = connection.BridgeCouldBeNull?.ConnectionType == ConnectionType.Interactive; ProductVariant? helloVariant = null, vendorVariant = null; string? vendorVersion = null; // a snapshot of the "version" value (IsScalar when present), so we can read it as a Version or a @@ -1226,15 +1231,17 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes // note: "mode" and "role" describe a *server*, so we only trust them when we're // talking to one directly; a proxy may answer HELLO from an arbitrary backend node // (envoy 1.39 does), and taking that at face value would flip us into cluster mode - // or mark the proxy as a replica + // or mark the proxy as a replica - and only from the interactive connection (above) case HelloField.Mode - when server.ServerType.SupportsAutoConfigure() + when helloFromInteractive + && server.ServerType.SupportsAutoConfigure() && iter.Value.TryParseScalar(&ServerTypeMetadata.TryParse, out ServerType serverType): server.ServerType = serverType; Log?.LogInformationAutoConfiguredHelloServerType(new(server), serverType); break; case HelloField.Role - when server.ServerType.SupportsAutoConfigure() + when helloFromInteractive + && server.ServerType.SupportsAutoConfigure() && iter.Value.TryParseScalar(&KnownRoleMetadata.TryParse, out bool isReplica): server.IsReplica = isReplica; server.RoleKnownFromHello = true; // so we don't need the key-based fallback probe diff --git a/src/StackExchange.Redis/ServerSelectionStrategy.cs b/src/StackExchange.Redis/ServerSelectionStrategy.cs index 2c79aa804..a3ea96c2d 100644 --- a/src/StackExchange.Redis/ServerSelectionStrategy.cs +++ b/src/StackExchange.Redis/ServerSelectionStrategy.cs @@ -322,6 +322,18 @@ internal bool OwnsAnySlot(ServerEndPoint server) return false; } + /// + /// Whether no slot has an owner, i.e. we hold no cluster topology to route by. + /// + /// + /// Derived from the map rather than tracked separately, which is sound because the map is allocated only by + /// , immediately before a write; the writers never store null and refuse + /// empty ranges (see ); and nothing ever clears an entry - + /// retirement declines to drop a server the map still references (). So a non-null + /// map always covers at least one slot. O(1); is the scan, for logging. + /// + internal bool IsSlotMapEmpty => map is null; + internal int CountCoveredSlots() { var arr = map; @@ -334,6 +346,11 @@ internal int CountCoveredSlots() internal void UpdateClusterRange(int fromInclusive, int toInclusive, ServerEndPoint server) { + // never allocate without writing (see IsSlotMapEmpty): SlotRange does not validate ordering, and the + // reply-driven paths build ranges straight from what the server sent - an inverted or out-of-range + // range is garbage, so ignore it rather than guess at what was meant (or index out of bounds) + if (fromInclusive < 0 || toInclusive >= RedisClusterSlotCount || fromInclusive > toInclusive) return; + var arr = MapForMutation(); for (int i = fromInclusive; i <= toInclusive; i++) { diff --git a/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs index 1f6e34de5..1932824dc 100644 --- a/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs @@ -24,6 +24,9 @@ protected override TypedRedisValue ClusterSlots(RedisClient client, in RedisRequ protected override TypedRedisValue ClusterNodes(RedisClient client, in RedisRequest request) => refuseNodes ? TypedRedisValue.Error("ERR command is not allowed") : base.ClusterNodes(client, request); + + // a proxy that hides the shards accepts cross-slot commands itself + public override bool CheckCrossSlot => false; } [Fact] @@ -71,5 +74,11 @@ public async Task AllClusterCommandsRefusedIsQuiet() // ...and a later explicit reconfigure stays quiet too Assert.True(await conn.ReconfigureAsync("test")); Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + + // with no slot map there is nothing to route by, so we must not *route* as a cluster - in particular, + // cross-slot commands go to the proxy rather than being refused client-side (#3284) + Assert.NotEmpty(logger.Matching("no slot map is available; routing as standalone")); + Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + await conn.GetDatabase().StringGetAsync(["a", "b"]); } } diff --git a/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs new file mode 100644 index 000000000..c5b827969 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs @@ -0,0 +1,99 @@ +using System; +using System.Threading.Tasks; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A proxied deployment can answer HELLO with mode: cluster while INFO says +/// redis_mode:standalone and every CLUSTER subcommand is refused (Azure Managed Redis with the +/// Enterprise clustering policy; see #3284). Such a server gives the client no slot map, so it must not be +/// treated as a cluster - in particular, cross-slot commands must keep working, including after a reconnect +/// of the subscription connection alone, whose handshake is HELLO without the INFO that follows +/// on the interactive connection. +/// +public class HelloClusterModeUnitTests(ITestOutputHelper log) +{ + private sealed class HelloSaysClusterServer(ITestOutputHelper log) : InProcessTestServer(log) + { + protected override string HelloModeValue => "cluster"; + + protected override TypedRedisValue ClusterSlots(RedisClient client, in RedisRequest request) + => TypedRedisValue.Error("ERR command is not allowed"); + + protected override TypedRedisValue ClusterNodes(RedisClient client, in RedisRequest request) + => TypedRedisValue.Error("ERR command is not allowed"); + } + + [Theory] + [InlineData(RedisProtocol.Resp2)] + [InlineData(RedisProtocol.Resp3)] + public async Task CrossSlotSurvivesSubscriptionReconnect(RedisProtocol protocol) + { + using var server = new HelloSaysClusterServer(log); // ServerType stays Standalone: INFO says so + + var config = server.GetClientConfig(defaultOnly: true); + config.Protocol = protocol; + config.AllowSimulateConnectionFailure = true; + config.AbortOnConnectFail = false; + var logger = new CapturingLogger(); + config.LoggerFactory = logger; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var db = conn.GetDatabase(); + + RedisKey[] crossSlot = ["a", "b"]; // different hash slots + await db.StringGetAsync(crossSlot); // sanity: fine on a fresh connection + Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + + var endpoint = conn.GetServerSnapshot()[0]; + await WaitUntilAsync(() => endpoint.IsSubscriberConnected, "subscriber connected"); + + // drop *only* the subscription connection, as the server did in #3284; the reconnect is too quick + // to observe by polling, so watch for the restored event instead + var restored = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + conn.ConnectionRestored += (_, e) => + { + if (e.ConnectionType == ConnectionType.Subscription) restored.TrySetResult(true); + }; + log.WriteLine("dropping the subscription connection"); + endpoint.SimulateConnectionFailure(SimulatedFailureType.AllSubscription); + await restored.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await WaitUntilAsync(() => endpoint.IsSubscriberConnected, "subscriber reconnected"); + + // the restored connection triggers a fire-and-forget reconfigure (and so can a server-type change); + // ReconfigureAsync declines while one is in flight, so insist on one that actually ran + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!await conn.ReconfigureAsync("after subscription reconnect")) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException("Timed out waiting for a reconfigure to run"); + await Task.Delay(50); + } + + try + { + // the user-visible symptom from #3284: cross-slot commands refused client-side + for (int i = 0; i < 5; i++) + { + await db.StringGetAsync(crossSlot); + await Task.Delay(100); + } + Assert.Equal(ServerType.Standalone, endpoint.ServerType); + Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + } + finally + { + log.WriteLine(logger.All); + } + } + + private static async Task WaitUntilAsync(Func condition, string what) + { + var deadline = DateTime.UtcNow.AddSeconds(10); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException("Timed out waiting for: " + what); + await Task.Delay(50); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/SlotMapUnitTests.cs b/tests/StackExchange.Redis.Tests/SlotMapUnitTests.cs new file mode 100644 index 000000000..3ab329e0d --- /dev/null +++ b/tests/StackExchange.Redis.Tests/SlotMapUnitTests.cs @@ -0,0 +1,33 @@ +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// is derived from whether the map has been allocated, which is +/// only sound if nothing can allocate it without writing an owner into it (see #3284/#3285). +/// +public class SlotMapUnitTests(ITestOutputHelper log) +{ + [Fact] + public async Task RejectedRangesDoNotAllocateTheMap() + { + using var server = new InProcessTestServer(log); + await using var conn = await ConnectionMultiplexer.ConnectAsync(server.GetClientConfig(withPubSub: false, defaultOnly: true)); + var strategy = conn.ServerSelectionStrategy; + var endpoint = conn.GetServerSnapshot()[0]; + + Assert.True(strategy.IsSlotMapEmpty); // standalone: nothing has been applied + Assert.Equal(0, strategy.CountCoveredSlots()); + + strategy.UpdateClusterRange(5, 3, endpoint); // inverted + strategy.UpdateClusterRange(-1, 3, endpoint); // below range + strategy.UpdateClusterRange(16380, 16384, endpoint); // above range + Assert.True(strategy.IsSlotMapEmpty); + Assert.Equal(0, strategy.CountCoveredSlots()); + + strategy.UpdateClusterRange(3, 5, endpoint); + Assert.False(strategy.IsSlotMapEmpty); + Assert.Equal(3, strategy.CountCoveredSlots()); + } +} diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index a0862ff86..5289b8791 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -675,7 +675,7 @@ protected virtual TypedRedisValue Hello(RedisClient client, in RedisRequest requ span[index++] = TypedRedisValue.BulkString("id"); span[index++] = TypedRedisValue.Integer(client.Id); span[index++] = TypedRedisValue.BulkString("mode"); - span[index++] = TypedRedisValue.BulkString(ServerModeValue); + span[index++] = TypedRedisValue.BulkString(HelloModeValue); span[index++] = TypedRedisValue.BulkString("role"); span[index++] = TypedRedisValue.BulkString(GetPrimaryOf(client?.Node) is null ? "master" : "replica"); span[index++] = TypedRedisValue.BulkString("modules"); @@ -699,6 +699,12 @@ protected virtual TypedRedisValue Hello(RedisClient client, in RedisRequest requ /// protected virtual IReadOnlyList> HelloExtraFields => []; + /// + /// The mode reported by HELLO; by default the same as INFO's redis_mode, but a + /// proxy can disagree with itself (see #3284), so tests can override this independently. + /// + protected virtual string HelloModeValue => ServerModeValue; + [RedisCommand(-3)] protected virtual TypedRedisValue Sadd(RedisClient client, in RedisRequest request) { From 10caa393ef9bea0819c09dfe10d7d5cce23982cc Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 22 Oct 2026 13:46:09 +0100 Subject: [PATCH 10/22] Fail connection-dropped commands with RedisConnectionException, never raw or cancelled (#3283) (#3287) When a connection failed, most queued commands failed with RedisConnectionException but a few received the raw transport exception (IOException, or the RESPite writer's InvalidOperationException) or a TaskCanceledException with no token in play. Three sources, all confirmed with the reporter's repro: - RecordConnectionFailed builds its detailed RedisConnectionException only for the bridge's current connection and only on the first report; a repeat report (the write path and the read loop both reporting), or one against a connection the bridge has already replaced, drained the queue with the raw inner exception - or with null, which the result box turns into a cancellation (every EOF path in the read loop reports with no exception). - The write-lock catch fails the message properly and then reports the connection, which drained the same still-queued message again with the raw exception; last write wins. - FailDueToNoConnection, TimedOutBeforeWrite and the fail-fast bin completed the message via Cancel(); ExecuteAsyncImpl then races to fault the task with the real exception, and the box's thread-pool continuation sometimes wins. A batch has no caller translating the result at all. Fix: the drain always uses a RedisConnectionException (wrapping whatever it was given, with the failure type), and the bridge's refusals set the exception the caller path would have built instead of cancelling. Message.Cancel itself is unchanged; transactions rely on it for aborted conditions. ConnectionFailureExceptionTests covers the drain (null and IOException inner, on a bridge-less connection, i.e. the non-current path) and the no-connection refusal observed at the message's own task. Reporter's repro, 90 s: 884 TaskCanceledException + 415 InvalidOperationException before; none after. Fixes #3283 --- src/StackExchange.Redis/PhysicalBridge.cs | 22 +++--- src/StackExchange.Redis/PhysicalConnection.cs | 8 ++ .../ConnectionFailureExceptionTests.cs | 78 +++++++++++++++++++ 3 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/ConnectionFailureExceptionTests.cs diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 7cf39edeb..eb81a8170 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -193,17 +193,18 @@ private WriteResult QueueOrFailMessage(Message message) } // Anything else goes in the bin - we're just not ready for you yet - message.Cancel(); - Multiplexer.OnMessageFaulted(message, null); - message.Complete(null); - return WriteResult.NoConnectionAvailable; + return FailDueToNoConnection(message); } private WriteResult FailDueToNoConnection(Message message) { - message.Cancel(); - Multiplexer.OnMessageFaulted(message, null); - message.Complete(null); + // fail with the exception the caller would otherwise construct from the result, rather than Cancel(): + // a cancelled box surfaces as TaskCanceledException, and although ExecuteAsyncImpl then tries to + // fault the task with the real exception, the box's continuation can get there first (#3283) - and a + // batch has no caller translating the result at all + var ex = ExceptionFactory.NoConnectionAvailable(Multiplexer, message, ServerEndPoint); + Multiplexer.OnMessageFaulted(message, ex); + message.SetExceptionAndComplete(ex, null); return WriteResult.NoConnectionAvailable; } @@ -1368,9 +1369,10 @@ public bool HasPendingCallerFacingItems() private WriteResult TimedOutBeforeWrite(Message message) { - message.Cancel(); - Multiplexer.OnMessageFaulted(message, null); - message.Complete(null); + // as FailDueToNoConnection: a real exception, not Cancel() + var ex = ExceptionFactory.Timeout(Multiplexer, null, message, ServerEndPoint, WriteResult.TimeoutBeforeWrite, this); + Multiplexer.OnMessageFaulted(message, ex); + message.SetExceptionAndComplete(ex, null); return WriteResult.TimeoutBeforeWrite; } diff --git a/src/StackExchange.Redis/PhysicalConnection.cs b/src/StackExchange.Redis/PhysicalConnection.cs index c960e4230..9e5eab6a1 100644 --- a/src/StackExchange.Redis/PhysicalConnection.cs +++ b/src/StackExchange.Redis/PhysicalConnection.cs @@ -696,6 +696,14 @@ void AddData(string? lk, string? sk, string? v) } var ex = innerException is RedisException ? innerException : outerException; + if (ex is not RedisException) + { + // the detailed exception above is built once, for the bridge's *current* connection; a repeat + // report for the same connection, or one the bridge has already replaced, must still fail + // whatever is queued - and with a RedisConnectionException, not the raw transport fault, nor + // (when there is no exception at all, e.g. EOF) a TaskCanceledException nobody asked for (#3283) + ex = new RedisConnectionException(failureType, CommandFlags.None, failureType + " on " + ToString(), innerException); + } nextMessage = Interlocked.Exchange(ref _awaitingToken, null); if (nextMessage is not null) diff --git a/tests/StackExchange.Redis.Tests/ConnectionFailureExceptionTests.cs b/tests/StackExchange.Redis.Tests/ConnectionFailureExceptionTests.cs new file mode 100644 index 000000000..79a8d84e1 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ConnectionFailureExceptionTests.cs @@ -0,0 +1,78 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A command that fails because its connection failed must fail with a +/// carrying the transport fault as its inner exception: never the raw transport exception, and never a +/// when no caller token was involved (#3283). +/// +public class ConnectionFailureExceptionTests(ITestOutputHelper log) +{ + private static (Message Message, Task Task) CreateTrackedPing() + { + var message = Message.Create(-1, CommandFlags.None, RedisCommand.PING); + var box = TaskResultBox.Create(out var tcs, null); + message.SetSource(box, ResultProcessor.DemandOK); + return (message, tcs.Task); + } + + private static async Task AwaitFailure(Task task) + { + var completed = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(10))); + Assert.Same(task, completed); + return await Assert.ThrowsAnyAsync(() => task); + } + + [Theory] + [InlineData(false)] // EOF-style report: no exception at all (used to surface as TaskCanceledException) + [InlineData(true)] // transport fault (used to surface raw when the detailed report was not built) + public async Task DrainedMessagesFailWithRedisConnectionException(bool withInner) + { + // a dummy connection has no bridge, so it is never the bridge's *current* connection and the detailed + // first-report exception is not built: this is exactly the path that handed out raw/null exceptions + using var stream = new MemoryStream(); + using var connection = PhysicalConnection.Dummy(stream); + var (message, task) = CreateTrackedPing(); + connection.EnqueueInsideWriteLock(message, enforceMuxer: false); + + var inner = withInner ? new IOException("simulated transport fault") : null; + connection.RecordConnectionFailed(ConnectionFailureType.SocketClosed, inner); + + var ex = await AwaitFailure(task); + log.WriteLine(ex.ToString()); + var conn = Assert.IsType(ex); + Assert.Equal(ConnectionFailureType.SocketClosed, conn.FailureType); + Assert.Same(inner, conn.InnerException); + } + + [Fact] + public async Task NoConnectionFailsWithRedisConnectionExceptionNotCancellation() + { + using var server = new InProcessTestServer(log); + var config = server.GetClientConfig(withPubSub: false, defaultOnly: true); + config.BacklogPolicy = BacklogPolicy.FailFast; + config.AbortOnConnectFail = false; + config.AllowSimulateConnectionFailure = true; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var endpoint = conn.GetServerSnapshot()[0]; + var bridge = endpoint.GetBridge(ConnectionType.Interactive)!; + + conn.AllowConnect = false; + endpoint.SimulateConnectionFailure(SimulatedFailureType.All); + Assert.False(conn.IsConnected); + + // write at the bridge, so we observe the message's own outcome rather than the caller's translation of + // the write result (which is what hid this for single commands - most of the time) + var (message, task) = CreateTrackedPing(); + Assert.Equal(WriteResult.NoConnectionAvailable, await bridge.TryWriteAsync(message, isReplica: false)); + + var ex = await AwaitFailure(task); + log.WriteLine(ex.ToString()); + Assert.IsType(ex); + Assert.False(task.IsCanceled); + } +} From 53aab24a7df66cda07ad2c07f15ff6136221f7c8 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 22 Oct 2026 14:50:27 +0100 Subject: [PATCH 11/22] Tests: tolerate clock skew on absolute hash-field expiries; wait for quiescence before asserting backlog counters (#3288) Two of the recurring Windows CI flakes. HashFieldTests' DateTime-based expiries are computed on the client clock and read back on the server's (in WSL), and the TTL came back as 7200001ms against a hard 7200000 ceiling: five tests in one run. BacklogTests.TotalOutstandingIncludesBacklogQueue asserted TotalOutstanding == 0 the instant the last backlogged ping completed, while the reconnect's handshake traffic was still in flight (seen as 7 and 1; six failures in the last twelve days). --- .../StackExchange.Redis.Tests/BacklogTests.cs | 6 +++ .../HashFieldTests.cs | 41 +++++++++++-------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/tests/StackExchange.Redis.Tests/BacklogTests.cs b/tests/StackExchange.Redis.Tests/BacklogTests.cs index ad30791ca..362945e1c 100644 --- a/tests/StackExchange.Redis.Tests/BacklogTests.cs +++ b/tests/StackExchange.Redis.Tests/BacklogTests.cs @@ -472,6 +472,12 @@ public async Task TotalOutstandingIncludesBacklogQueue() Log("Test: Checking reconnected"); Assert.True(conn.IsConnected); + // lastPing completing proves the backlog drained, not that every connection is idle: the reconnect's own + // handshake traffic (and the subscription connection's) can still be awaiting replies at this instant, + // and TotalOutstanding counts those too (seen on CI as 7 and 1); so wait for quiescence rather than assert it + Log("Test: awaiting quiescence"); + await UntilConditionAsync(TimeSpan.FromSeconds(5), () => conn.GetCounters().TotalOutstanding == 0).ForAwait(); + Log("Test: asserting reconnected counters"); var reconnectedServerCounters = server.GetCounters(); var reconnectedConnCounters = conn.GetCounters(); diff --git a/tests/StackExchange.Redis.Tests/HashFieldTests.cs b/tests/StackExchange.Redis.Tests/HashFieldTests.cs index 2bb98eb85..50fa49a71 100644 --- a/tests/StackExchange.Redis.Tests/HashFieldTests.cs +++ b/tests/StackExchange.Redis.Tests/HashFieldTests.cs @@ -11,6 +11,11 @@ namespace StackExchange.Redis.Tests; [RunPerProtocol] public class HashFieldTests(ITestOutputHelper output, SharedConnectionFixture fixture) : TestBase(output, fixture) { + // absolute expiries (the DateTime overloads) are computed against the client clock and read back against the + // server's; on the Windows CI runner the server lives in WSL, whose clock can sit a few ms off the host's, and + // the TTL then lands just above the nominal 2h (seen: 7200001ms). Relative (TimeSpan) expiries use one clock only + private const double ClockSkewToleranceMs = 1000; + private readonly DateTime nextCentury = new DateTime(2101, 1, 1, 0, 0, 0, DateTimeKind.Utc); private readonly TimeSpan oneYearInMs = TimeSpan.FromMilliseconds(31536000000); @@ -319,7 +324,7 @@ public void HashFieldGetAndSetExpiry() fieldResult = db.HashFieldGetAndSetExpiry(hashKey, "f1", DateTime.Now.AddMinutes(120)); Assert.Equal(1, fieldResult); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing persist fieldResult = db.HashFieldGetAndSetExpiry(hashKey, "f1", persist: true); @@ -340,8 +345,8 @@ public void HashFieldGetAndSetExpiry() fieldResults = db.HashFieldGetAndSetExpiry(hashKey, fields, DateTime.Now.AddMinutes(120)); Assert.Equal(values, fieldResults); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with persist fieldResults = db.HashFieldGetAndSetExpiry(hashKey, fields, persist: true); @@ -369,7 +374,7 @@ public async Task HashFieldGetAndSetExpiryAsync() fieldResult = await db.HashFieldGetAndSetExpiryAsync(hashKey, "f1", DateTime.Now.AddMinutes(120)); Assert.Equal(1, fieldResult); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing persist fieldResult = await db.HashFieldGetAndSetExpiryAsync(hashKey, "f1", persist: true); @@ -390,8 +395,8 @@ public async Task HashFieldGetAndSetExpiryAsync() fieldResults = await db.HashFieldGetAndSetExpiryAsync(hashKey, fields, DateTime.Now.AddMinutes(120)); Assert.Equal(values, fieldResults); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with persist fieldResults = await db.HashFieldGetAndSetExpiryAsync(hashKey, fields, persist: true); @@ -417,13 +422,13 @@ public void HashFieldSetAndSetExpiry() result = db.HashFieldSetAndSetExpiry(hashKey, "f1", 1, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with keepttl result = db.HashFieldSetAndSetExpiry(hashKey, "f1", 1, keepTtl: true); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with timespan result = db.HashFieldSetAndSetExpiry(hashKey, entries, TimeSpan.FromHours(1)); @@ -436,15 +441,15 @@ public void HashFieldSetAndSetExpiry() result = db.HashFieldSetAndSetExpiry(hashKey, entries, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with keepttl result = db.HashFieldSetAndSetExpiry(hashKey, entries, keepTtl: true); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with ExpireWhen.Exists db.KeyDelete(hashKey); @@ -481,13 +486,13 @@ public async Task HashFieldSetAndSetExpiryAsync() result = await db.HashFieldSetAndSetExpiryAsync(hashKey, "f1", 1, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with keepttl result = await db.HashFieldSetAndSetExpiryAsync(hashKey, "f1", 1, keepTtl: true); Assert.Equal(1, result); fieldTtl = db.HashFieldGetTimeToLive(hashKey, new RedisValue[] { "f1" })[0]; - Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtl, TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with timespan result = await db.HashFieldSetAndSetExpiryAsync(hashKey, entries, TimeSpan.FromHours(1)); @@ -500,15 +505,15 @@ public async Task HashFieldSetAndSetExpiryAsync() result = await db.HashFieldSetAndSetExpiryAsync(hashKey, entries, DateTime.Now.AddMinutes(120)); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing multiple fields with keepttl result = await db.HashFieldSetAndSetExpiryAsync(hashKey, entries, keepTtl: true); Assert.Equal(1, result); fieldTtls = db.HashFieldGetTimeToLive(hashKey, fields); - Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); - Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds); + Assert.InRange(fieldTtls[0], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); + Assert.InRange(fieldTtls[1], TimeSpan.FromMinutes(119).TotalMilliseconds, TimeSpan.FromHours(2).TotalMilliseconds + ClockSkewToleranceMs); // testing with ExpireWhen.Exists db.KeyDelete(hashKey); From cb09290e88ecc6bbe8986eeec849ed9af786c5e8 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 22 Oct 2026 14:51:07 +0100 Subject: [PATCH 12/22] Connection groups: a member answering LOADING triggers an immediate re-probe and re-select, like a breaker trip (#3289) The group's fast-path only reacted to ConnectionFailureType.CircuitBreaker. A server that is loading fails every reconnect during the handshake, so commands routed to it die in the backlog without a reply, and the breaker only observes replies - it can never be what moves traffic away. This is the shape of the recurring Windows CI failure of WithRetry_FailsOverBetweenGroupsOnLoading (six attempts in twelve days, always 20 x 5000ms backlog timeouts): when the heartbeat rather than the test's command met the first LOADING, nothing ever failed over. LOADING now joins CircuitBreaker in the fast-path; the health check still decides where traffic goes, so a member whose probe passes is kept (WithRetry_WhenFailoverNeverArrives_KeepsRetryingSameMember). New LoadingRerouteTests fails without the change. --- docs/Failover.md | 4 +- .../Availability/MultiGroupMultiplexer.cs | 16 ++-- .../MultiGroupTests/LoadingRerouteTests.cs | 78 +++++++++++++++++++ .../RetryTests/RetryEndToEndTests.cs | 7 +- 4 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/MultiGroupTests/LoadingRerouteTests.cs diff --git a/docs/Failover.md b/docs/Failover.md index 58a4c43eb..4275a1bd4 100644 --- a/docs/Failover.md +++ b/docs/Failover.md @@ -380,7 +380,7 @@ ConnectionGroupMember[] members = [ ]; ``` -To disable periodic checking for the whole group, set `MultiGroupOptions.HealthCheckInterval` to `TimeSpan.MaxValue`; the group is then only re-evaluated in response to connection events (such as a tripped circuit-breaker). +To disable periodic checking for the whole group, set `MultiGroupOptions.HealthCheckInterval` to `TimeSpan.MaxValue`; the group is then only re-evaluated in response to connection events (a tripped circuit-breaker, or a member answering `LOADING`). ### Built-in Probes @@ -495,6 +495,8 @@ Where health checks *actively* probe each member on a timer, a **circuit breaker When a circuit breaker trips, it shuts the underlying physical connection down with `ConnectionFailureType.CircuitBreaker`. The connection then reconnects as usual, and - in a connection group - the health check and member-selection logic route traffic to other members until the affected member recovers. A circuit breaker is evaluated **per connection**, so its state is scoped to exactly the connection whose health it is measuring; a reconnect starts from a clean slate - but breaking the connection is sufficient for the group to notice non-availability. +A member that answers `-LOADING` (a server still loading its dataset after a restart) is treated the same way by a connection group: the connection is dropped with `ConnectionFailureType.Loading`, the group re-probes and re-selects immediately, and traffic moves to another member rather than queuing against one whose every reconnect will fail with the same error until the load completes. The health check brings the member back once it passes. + ### Configuring a Circuit Breaker for a Group Circuit breakers are configured for all members via `MultiGroupOptions`, alongside the health check. The setting flows into every member connection: diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs index a426fdb78..82b8ebc4d 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -993,13 +993,17 @@ private static bool RemoveHandler(ref T? field, T? value) where T : Delegate } } - private int _circuitBreakerDebounce = 0; + private int _rerouteDebounce = 0; private void OnMemberConnectionFailed(object? sender, ConnectionFailedEventArgs e) { - // deliberately scoped to this one failure type; re-probe and re-select promptly rather - // than waiting for the next poll tick, so traffic routes away from the dropped member - if (e.FailureType is ConnectionFailureType.CircuitBreaker - && Interlocked.CompareExchange(ref _circuitBreakerDebounce, 1, 0) is 0) + // deliberately scoped to these failure types; re-probe and re-select promptly rather than + // waiting for the next poll tick, so traffic routes away from the dropped member. A breaker + // trip is the member's own traffic telling us it is unstable; LOADING is the server telling + // us outright that it cannot serve yet (and its handshake will keep failing the same way + // until it can, so nothing routed there in the meantime gets a reply - the breaker only ever + // sees replies, so it cannot be what moves us). Ordinary socket drops are left to reconnect + if (e.FailureType is ConnectionFailureType.CircuitBreaker or ConnectionFailureType.Loading + && Interlocked.CompareExchange(ref _rerouteDebounce, 1, 0) is 0) { GetMember(sender as ConnectionMultiplexer)?.SetUnhealthy(); _ = Task.Run(async () => @@ -1014,7 +1018,7 @@ private void OnMemberConnectionFailed(object? sender, ConnectionFailedEventArgs } finally { - Volatile.Write(ref _circuitBreakerDebounce, 0); + Volatile.Write(ref _rerouteDebounce, 0); } }); } diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/LoadingRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/LoadingRerouteTests.cs new file mode 100644 index 000000000..e03a26d03 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/LoadingRerouteTests.cs @@ -0,0 +1,78 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +[RunPerProtocol] +public class LoadingRerouteTests(ITestOutputHelper log) : TestBase(log) +{ + // A member answering LOADING must steer the group away promptly, via the same ConnectionFailed fast-path + // as a circuit-breaker trip. The breaker cannot be what moves us here: once the server is loading, every + // reconnect fails during the handshake, so later commands die in the backlog without a reply, and the + // breaker only ever observes replies. As with the breaker test, the poll interval is enormous so that the + // fast-path is the only thing that can reroute within the test window, and the probe holds A down so the + // reroute is deterministic. + [Fact] + public async Task LoadingMember_ReroutesAwayPromptly() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + + var probe = new ControllableProbe(); + + ConnectionGroupMember[] members = + [ + new(serverA.GetClientConfig(), "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target + ]; + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.FromMinutes(30), // enormous: the fast-path is what reroutes us + HealthCheck = new HealthCheck.Builder + { + Probe = probe, + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + var typed = Assert.IsType(conn); + + var loadingEvents = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + typed.ConnectionFailed += (_, e) => + { + Log($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); + if (e.FailureType == ConnectionFailureType.Loading) + { + loadingEvents.TrySetResult(true); + } + }; + + await GroupWait.AssertConnectedAsync(conn); + Assert.Same(members[0], conn.ActiveMember); + + // knock A into LOADING and hold it down; the next command to A is what surfaces the failure + probe.MarkDown(alpha); + serverA.IsLoading = true; + var fault = await Assert.ThrowsAnyAsync(() => conn.GetDatabase().PingAsync()); + Log($"observed fault: {fault.GetType().Name}: {fault.Message}"); + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (DateTime.UtcNow < deadline && ReferenceEquals(conn.ActiveMember, members[0])) + { + await Task.Delay(50, TestContext.Current.CancellationToken); + } + Assert.Same(members[1], conn.ActiveMember); + + var completed = await Task.WhenAny(loadingEvents.Task, Task.Delay(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken)); + Assert.True(ReferenceEquals(completed, loadingEvents.Task), "expected a ConnectionFailed event with FailureType == Loading"); + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs index aa9614027..0d49de0c8 100644 --- a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -225,8 +225,9 @@ public async Task WithRetry_FailsOverBetweenGroupsOnLoading() } // The unhappy path of the failover threshold: the attempt count says "wait for a failover now", but no - // failover ever comes (the member stays nominally healthy - a couple of LOADING replies are not enough - // to trip the default breaker). We wait out FailoverDelay and then carry on retrying the original + // failover ever comes (the member stays nominally healthy: a couple of LOADING replies are not enough + // to trip the default breaker, and the re-probe they trigger finds the member healthy, so the group + // keeps it). We wait out FailoverDelay and then carry on retrying the original // member rather than giving up, and the group never moves. [Fact] public async Task WithRetry_WhenFailoverNeverArrives_KeepsRetryingSameMember() From 74c52bf817391f34fe55b4a6befab202d3d29520 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 22 Oct 2026 14:51:24 +0100 Subject: [PATCH 13/22] IBatch: complete commands refused by a poisoned connection; divert a batch's tail on reconnect (#3286) * Complete a message refused by a poisoned connection; divert a batch's tail on reconnect (#3282) Since #3205 a connection whose write faulted is poisoned and refuses further writes, so that no reply can be matched to the wrong caller. That refusal returned WriteFailure without completing the message. Single commands were unaffected, as ExecuteAsyncImpl/ExecuteSyncImpl translate a non-success result into an exception for the caller; but RedisBatch.Execute writes a list via PhysicalBridge.TryEnqueue, which does not act on the result, so once one message of a batch faulted the connection, the rest of that batch was neither queued nor completed and its tasks were awaited forever (3.2.0 to 3.4.5). - WriteMessageToServerInsideWriteLock: the IsWriteFaulted refusal now fails and completes the message, as every other refusal to write already does (FailDueToNoConnection, TimedOutBeforeWrite, QueueOrFailMessage, the write-lock catch, the transaction sub-command loop). - TryEnqueue(list): once NeedsReconnect is set part-way through the batch, the remaining messages take the same path a single command would at the top of TryWriteSync - backlog or fail, per policy - instead of being written to a connection known to be dead. BatchWriteFailureTests injects a write failure mid-batch under both backlog policies and asserts that every task completes and that any successful result is real. The reporter's repro (400 batches in flight, CLIENT KILL in a loop) hung within 13 s on main and ran 6.6 M batches in 120 s with this change. Fixes #3282 * Batch: route the tail to the backlog when the held connection is poisoned, not just while the reconnect flag is set The reconnect flag is cleared as soon as TryConnect starts a replacement connection, on another thread; a batch still holding the previous PhysicalConnection could then write its remaining messages to it and have them refused with WriteFailure. Seen on the Windows CI runner (2 of 3 attempts); reproduces locally 12/25 pinned to two cores. Also make the test's fail-fast variant wait for the reconnect before its verification read, which is the other way the two-core run failed. --- src/StackExchange.Redis/PhysicalBridge.cs | 17 +++- .../BatchWriteFailureTests.cs | 86 +++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index eb81a8170..d1cb8b5b1 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -811,6 +811,17 @@ internal bool TryEnqueue(List messages, bool isReplica) } foreach (var message in messages) { + if (NeedsReconnect || physical.IsWriteFaulted) + { + // the connection went away part-way through the batch (typically an earlier message's write + // faulted it); give the remainder the treatment a single command gets at the top of + // TryWriteSync - backlog or fail, per policy - rather than writing to a dead connection (#3282). + // Both flags matter: the reconnect flag is cleared as soon as a replacement connection starts + // (TryConnect, on another thread), but the connection we are still holding stays poisoned + QueueOrFailMessage(message); + continue; + } + // deliberately not taking a single lock here; we don't care if // other threads manage to interleave - in fact, it would be desirable // (to avoid a batch monopolising the connection) @@ -1655,7 +1666,11 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne if (connection.IsWriteFaulted) { - // never add to the response queue of a connection whose queue we no longer trust + // never add to the response queue of a connection whose queue we no longer trust - but, like every + // other refusal to write, complete the message *here*: a caller writing a list (a batch) does not + // act on the result, and a message that is neither queued nor completed is awaited forever (#3282) + message.Fail(ConnectionFailureType.SocketFailure, null, "an earlier write on this connection failed; this command was not written", Multiplexer); + message.Complete(connection); return WriteResult.WriteFailure; } diff --git a/tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs b/tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs new file mode 100644 index 000000000..5ca7a2789 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs @@ -0,0 +1,86 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A write failure part-way through an poisons the connection; the batch's remaining +/// messages must still complete - faulted, or backlogged and delivered after the reconnect - never be left +/// awaiting a reply that cannot come (#3282). +/// +public class BatchWriteFailureTests(ITestOutputHelper log) +{ + private sealed class ThrowingMessage(Exception toThrow) : Message(-1, CommandFlags.None, RedisCommand.PING) + { + public override int ArgCount => 0; + + protected override void WriteImpl(in MessageWriter writer) => throw toThrow; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task EveryBatchCommandCompletesAfterMidBatchWriteFailure(bool queueWhileDisconnected) + { + using var server = new InProcessTestServer(log); + var config = server.GetClientConfig(withPubSub: false, defaultOnly: true); + config.BacklogPolicy = queueWhileDisconnected ? BacklogPolicy.Default : BacklogPolicy.FailFast; + config.AbortOnConnectFail = false; + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var db = conn.GetDatabase(); + RedisKey key = nameof(EveryBatchCommandCompletesAfterMidBatchWriteFailure); + await db.KeyDeleteAsync(key); + + var batch = (RedisBatch)db.CreateBatch(); + var before = batch.StringIncrementAsync(key); + var boom = batch.ExecuteAsync(new ThrowingMessage(new InvalidOperationException("simulated WriteImpl failure")), ResultProcessor.DemandOK, false); + var after1 = batch.StringIncrementAsync(key); + var after2 = batch.StringIncrementAsync(key); + batch.Execute(); + + Task[] all = [before, boom, after1, after2]; + var everything = Task.WhenAll(all); + if (await Task.WhenAny(everything, Task.Delay(TimeSpan.FromSeconds(10))) != everything) + { + Assert.Fail("hung: " + string.Join(", ", all.Select((t, i) => $"cmd{i}={t.Status}"))); + } + // individual faults are inspected below + + for (int i = 0; i < all.Length; i++) + { + log.WriteLine($"cmd{i}: {all[i].Status} {all[i].Exception?.GetBaseException().GetType().Name} {all[i].Exception?.GetBaseException().Message}"); + } + Assert.All(all, t => Assert.True(t.IsCompleted)); + Assert.True(boom.IsFaulted); + // "before" was written and then torn down with the connection: answered first, or failed with it. + // The tail is backlogged and delivered on the new connection, or refused (cancelled before the write, + // or faulted at the poisoned connection) - never left pending. Whatever succeeded must be real: the + // counter reflects every INCR the server ran, which is at least every one reported as successful + Task[] increments = [before, after1, after2]; + if (queueWhileDisconnected) + { + Assert.True(after1.IsCompletedSuccessfully, after1.Status.ToString()); + Assert.True(after2.IsCompletedSuccessfully, after2.Status.ToString()); + } + else + { + Assert.All([after1, after2], t => Assert.True( + t.IsCompletedSuccessfully || t.IsCanceled || t.Exception?.GetBaseException() is RedisConnectionException, t.Status.ToString())); + } + long succeeded = increments.Count(t => t.IsCompletedSuccessfully); + + // under fail-fast the tail is refused rather than backlogged, so nothing above waited for the reconnect; + // the verification read must not fail for the same reason + var reconnected = Task.Run(async () => + { + while (!conn.IsConnected) await Task.Delay(50); + }); + if (await Task.WhenAny(reconnected, Task.Delay(TimeSpan.FromSeconds(10))) != reconnected) + { + Assert.Fail("did not reconnect"); + } + Assert.InRange((long)await db.StringGetAsync(key), succeeded, increments.Length); + } +} From a07aabbaee785469c6ac570d2cf95486cbb877ee Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Thu, 22 Oct 2026 17:37:34 +0100 Subject: [PATCH 14/22] Tests: don't race the connect's own reconfigures in AllClusterCommandsRefusedIsQuiet (#3290) A freshly connected multiplexer fires fire-and-forget reconfigures of its own (connection-restored per physical connection, autoconfigure field changes), and ReconfigureAsync returns false while one is in flight. The test asserted on that return value directly, so it lost the race on the Windows runner (main @ 74c52bf8, net10.0/RESP3). Hoist the retry loop HelloClusterModeUnitTests already used for this into Poll.ReconfigureAsync and use it in both tests; dump the captured log after the assertions so a future failure shows what was actually logged. --- .../ClusterCommandsRefusedUnitTests.cs | 33 ++++++++++++------- .../HelloClusterModeUnitTests.cs | 7 +--- .../StackExchange.Redis.Tests/Helpers/Poll.cs | 20 +++++++++++ 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs index 1932824dc..303714560 100644 --- a/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ClusterCommandsRefusedUnitTests.cs @@ -65,20 +65,29 @@ public async Task AllClusterCommandsRefusedIsQuiet() config.LoggerFactory = logger; await using var conn = await ConnectionMultiplexer.ConnectAsync(config); - log.WriteLine(logger.All); - Assert.Empty(logger.Matching("Encountered error while updating cluster config")); - Assert.NotEmpty(logger.Matching("CLUSTER NODES was refused")); + try + { + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + Assert.NotEmpty(logger.Matching("CLUSTER NODES was refused")); - await conn.GetDatabase().PingAsync(); + await conn.GetDatabase().PingAsync(); - // ...and a later explicit reconfigure stays quiet too - Assert.True(await conn.ReconfigureAsync("test")); - Assert.Empty(logger.Matching("Encountered error while updating cluster config")); + // ...and a later explicit reconfigure stays quiet too; the connect's own follow-on reconfigures + // (connection restored, autoconfigure changes) may still be in flight, and ReconfigureAsync declines + // while one is, so insist on a pass that actually ran + await Poll.ReconfigureAsync(conn, "test"); + Assert.Empty(logger.Matching("Encountered error while updating cluster config")); - // with no slot map there is nothing to route by, so we must not *route* as a cluster - in particular, - // cross-slot commands go to the proxy rather than being refused client-side (#3284) - Assert.NotEmpty(logger.Matching("no slot map is available; routing as standalone")); - Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); - await conn.GetDatabase().StringGetAsync(["a", "b"]); + // with no slot map there is nothing to route by, so we must not *route* as a cluster - in particular, + // cross-slot commands go to the proxy rather than being refused client-side (#3284) + Assert.NotEmpty(logger.Matching("no slot map is available; routing as standalone")); + Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + await conn.GetDatabase().StringGetAsync(["a", "b"]); + } + finally + { + // after the assertions, so a failure shows what the multiplexer actually logged + log.WriteLine(logger.All); + } } } diff --git a/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs index c5b827969..50336b9ab 100644 --- a/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs @@ -63,12 +63,7 @@ public async Task CrossSlotSurvivesSubscriptionReconnect(RedisProtocol protocol) // the restored connection triggers a fire-and-forget reconfigure (and so can a server-type change); // ReconfigureAsync declines while one is in flight, so insist on one that actually ran - var deadline = DateTime.UtcNow.AddSeconds(10); - while (!await conn.ReconfigureAsync("after subscription reconnect")) - { - if (DateTime.UtcNow > deadline) throw new TimeoutException("Timed out waiting for a reconfigure to run"); - await Task.Delay(50); - } + await Poll.ReconfigureAsync(conn, "after subscription reconnect"); try { diff --git a/tests/StackExchange.Redis.Tests/Helpers/Poll.cs b/tests/StackExchange.Redis.Tests/Helpers/Poll.cs index a5a0dc4ff..dd55d1d08 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/Poll.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/Poll.cs @@ -37,4 +37,24 @@ public static async Task UntilAsync(Func predicate, int timeoutMilli return false; } + + /// + /// Runs a reconfigure that is guaranteed to have actually executed, rather than been declined. + /// + /// + /// A freshly connected multiplexer fires its own fire-and-forget reconfigures (connection-restored for each + /// physical connection, autoconfigure field changes), and + /// returns false while one is in flight. A test that asserts on that return value races those, and + /// loses on a slow runner; this keeps asking until a pass runs under the caller's cause. + /// + public static async Task ReconfigureAsync(ConnectionMultiplexer conn, string cause, int timeoutMilliseconds = 10000, int pollMilliseconds = 50) + { + var cancellationToken = TestContext.Current.CancellationToken; + for (int i = 0; i <= timeoutMilliseconds / pollMilliseconds; i++) + { + if (await conn.ReconfigureAsync(cause).ConfigureAwait(false)) return; + await Task.Delay(pollMilliseconds, cancellationToken).ConfigureAwait(false); + } + throw new TimeoutException("Timed out waiting for a reconfigure to run: " + cause); + } } From 6cbaa3750cc7b7ce925402e47b29291228367a6d Mon Sep 17 00:00:00 2001 From: AsadIqbal-Software Engineer Date: Fri, 23 Oct 2026 13:48:52 +0500 Subject: [PATCH 15/22] Send primary-only commands to the primary when the caller prefers a replica (#3291) * Send primary-only commands to the primary when the caller prefers a replica KeyDelete, KeyExpire (milliseconds), LockExtend and LockRelease pick a server up front to check its features. With PreferReplica that picked a replica, and the primary-only message was then rejected with 'Command cannot be issued to a replica'. GetFeatures now upgrades the flags the same way Message does. Fixes #2821 * Mirror Message.SetPrimaryOnly exactly; explain the PEXPIRE expiry in the test --------- Co-authored-by: Marc Gravell --- src/StackExchange.Redis/RedisBase.cs | 7 +++++ tests/StackExchange.Redis.Tests/KeyTests.cs | 31 +++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/StackExchange.Redis/RedisBase.cs b/src/StackExchange.Redis/RedisBase.cs index 84981b71f..f49d43ce1 100644 --- a/src/StackExchange.Redis/RedisBase.cs +++ b/src/StackExchange.Redis/RedisBase.cs @@ -68,6 +68,13 @@ internal virtual Task ExecuteAsync(Message? message, ResultProcessor? p internal virtual RedisFeatures GetFeatures(in RedisKey key, CommandFlags flags, RedisCommand command, out ServerEndPoint? server) { + // the server picked here is the one the message is sent to, so a primary-only command has to pick the primary + // here as well, the same way Message.SetPrimaryOnly does when the message is created; otherwise PreferReplica + // picks a replica that then rejects it (DemandReplica is left alone so that it still fails the same way) + if (command.IsPrimaryOnly() && Message.GetPrimaryReplicaFlags(flags) != CommandFlags.DemandReplica) + { + flags = Message.SetPrimaryReplicaFlags(flags, CommandFlags.DemandMaster); + } server = multiplexer.SelectServer(command, flags, key); var version = server == null ? multiplexer.RawConfig.DefaultVersion : server.Version; return new RedisFeatures(version); diff --git a/tests/StackExchange.Redis.Tests/KeyTests.cs b/tests/StackExchange.Redis.Tests/KeyTests.cs index 322cbf00b..a8638be53 100644 --- a/tests/StackExchange.Redis.Tests/KeyTests.cs +++ b/tests/StackExchange.Redis.Tests/KeyTests.cs @@ -75,6 +75,37 @@ public async Task KeyTypeOfMissingKeyIsNone() // see #3156 Assert.Equal(RedisType.None, await db.KeyTypeAsync(key)); } + [Fact] + public async Task PrimaryOnlyCommandsWithPreferReplicaRunOnThePrimary() // see #2821 + { + await using var conn = Create(configuration: TestConfig.Current.PrimaryServerAndPort + "," + TestConfig.Current.ReplicaServerAndPort); + Assert.SkipUnless( + conn.GetEndPoints().Any(ep => conn.GetServer(ep).IsReplica), + "No replica in this configuration"); + + var db = conn.GetDatabase(); + var key = Me(); + RedisValue token = Guid.NewGuid().ToString(); + + // these look at the server's features before building the message, which used to pick the replica; + // the fractional-second expiry forces the PEXPIRE path, which is the one that checks features + db.StringSet(key, "abc"); + Assert.True(db.KeyExpire(key, TimeSpan.FromMilliseconds(100_500), CommandFlags.PreferReplica)); + Assert.True(db.KeyDelete(key, CommandFlags.PreferReplica)); + + db.StringSet(key, "abc"); + Assert.True(await db.KeyExpireAsync(key, TimeSpan.FromMilliseconds(100_500), CommandFlags.PreferReplica)); + Assert.True(await db.KeyDeleteAsync(key, CommandFlags.PreferReplica)); + + Assert.True(db.LockTake(key, token, TimeSpan.FromMinutes(1))); + Assert.True(db.LockExtend(key, token, TimeSpan.FromMinutes(1), CommandFlags.PreferReplica)); + Assert.True(await db.LockExtendAsync(key, token, TimeSpan.FromMinutes(1), CommandFlags.PreferReplica)); + Assert.True(db.LockRelease(key, token, CommandFlags.PreferReplica)); + + Assert.True(db.LockTake(key, token, TimeSpan.FromMinutes(1))); + Assert.True(await db.LockReleaseAsync(key, token, CommandFlags.PreferReplica)); + } + [Fact] public async Task Zeros() { From 946f13ae783ee1ed3b6dcdc768771868a2fa925b Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 23 Oct 2026 10:01:27 +0100 Subject: [PATCH 16/22] Give multi-message sub-commands and injected prefix commands a write time (#3292) (#3293) Sub-commands of an IMultiMessage (WATCH, condition reads, MULTI, the queued operations; also SCRIPT LOAD and PTTL) are written inside the lock without passing through SetEnqueued, as are the SELECT/READONLY/ASKING/PREPARE prefixes injected ahead of a command. With a zero write time the heartbeat sees them as overdue by the process uptime; since #2965 such no-box messages count as sync timeouts, and together with an idle connection that satisfies the dead-socket heuristic while the reply is still in flight. Sub-commands now inherit the parent's write time (sharing its budget, including backlog time); prefix commands are stamped when written, via a single helper replacing four identical blocks. --- src/StackExchange.Redis/Message.cs | 7 ++ src/StackExchange.Redis/PhysicalBridge.cs | 39 +++++---- .../MultiMessageWriteTimeTests.cs | 79 +++++++++++++++++++ 3 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/MultiMessageWriteTimeTests.cs diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index c6aceb60b..e6943cbcf 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -887,6 +887,13 @@ internal void SetWriteTime() .TickCount; // note this might be reset if we resend a message, cluster-moved etc; I'm OK with that } + /// + /// Sub-commands of an are written inside the lock without passing through + /// ; they take the parent's write time, so that they share its timeout budget + /// (including any time it spent in the backlog) rather than looking overdue by the process uptime. + /// + internal void InheritWriteTime(Message parent) => _writeTickCount = parent._writeTickCount; + private int _writeTickCount; public int GetWriteTime() => Volatile.Read(ref _writeTickCount); diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index d1cb8b5b1..3ad0afb28 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -865,6 +865,9 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message SelectDatabaseInsideWriteLock(physical, message); // need to switch database *before* the transaction foreach (var subCommand in multiMessage.GetMessages(physical)) { + // sub-commands do not pass through SetEnqueued (only the parent did, before the backlog); + // without a write time the heartbeat sees them as overdue by the process uptime (#3292) + if (!ReferenceEquals(subCommand, message)) subCommand.InheritWriteTime(message); result = WriteMessageToServerInsideWriteLock(physical, subCommand); if (result != WriteResult.Success) { @@ -1614,6 +1617,21 @@ private void OnInternalError(Exception exception, [CallerMemberName] string? ori Multiplexer.OnInternalError(exception, ServerEndPoint.EndPoint, ConnectionType, origin); } + /// + /// Writes a fire-and-forget command that is injected ahead of a caller's message (SELECT, READONLY, ASKING, PREPARE). + /// These never pass through , so the write time is stamped here; some of them are + /// shared static instances, so the stamp can be overwritten by another connection, but only ever with a later time, + /// which can make a pending prefix look younger than it is - never older (#3292). + /// + private void WritePrefixCommandInsideWriteLock(PhysicalConnection connection, Message prefix) + { + prefix.SetWriteTime(); + connection.EnqueueInsideWriteLock(prefix); + prefix.WriteTo(connection); + prefix.SetRequestSent(); + IncrementOpCount(); + } + private void SelectDatabaseInsideWriteLock(PhysicalConnection connection, Message message) { int db = message.Db; @@ -1622,10 +1640,7 @@ private void SelectDatabaseInsideWriteLock(PhysicalConnection connection, Messag var sel = connection.GetSelectDatabaseCommand(db, message); if (sel != null) { - connection.EnqueueInsideWriteLock(sel); - sel.WriteTo(connection); - sel.SetRequestSent(); - IncrementOpCount(); + WritePrefixCommandInsideWriteLock(connection, sel); } } } @@ -1644,10 +1659,7 @@ private void PrepareFieldSetInsideWriteLock(PhysicalConnection connection, Messa if (connection.TryAddPreparedFieldSet(fieldSet.Id)) { var prepare = fieldSet.CreatePrepareMessage(message.Db); - connection.EnqueueInsideWriteLock(prepare); - prepare.WriteTo(connection); - prepare.SetRequestSent(); - IncrementOpCount(); + WritePrefixCommandInsideWriteLock(connection, prepare); fieldSet.RegisterServer(ServerEndPoint, message.Db); } } @@ -1707,19 +1719,12 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne var readmode = connection.GetReadModeCommand(isPrimaryOnly); if (readmode != null) { - connection.EnqueueInsideWriteLock(readmode); - readmode.WriteTo(connection); - readmode.SetRequestSent(); - IncrementOpCount(); + WritePrefixCommandInsideWriteLock(connection, readmode); } } if (message.IsAsking) { - var asking = ReusableAskingCommand; - connection.EnqueueInsideWriteLock(asking); - asking.WriteTo(connection); - asking.SetRequestSent(); - IncrementOpCount(); + WritePrefixCommandInsideWriteLock(connection, ReusableAskingCommand); } } switch (cmd) diff --git a/tests/StackExchange.Redis.Tests/MultiMessageWriteTimeTests.cs b/tests/StackExchange.Redis.Tests/MultiMessageWriteTimeTests.cs new file mode 100644 index 000000000..4a1f0c408 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiMessageWriteTimeTests.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// The sub-commands of a transaction (and the SELECT/READONLY/ASKING prefixes written ahead of a command) are +/// added to the awaiting-response queue without going through the normal enqueue path. If they carry no write +/// time, the heartbeat sees them as overdue by the process uptime, and on a connection that has been idle for +/// a while that is enough to trip the dead-socket heuristic while the reply is still in flight; see #3292. +/// +[Collection(NonParallelCollection.Name)] +public class MultiMessageWriteTimeTests(ITestOutputHelper output) : TestBase(output) +{ + [Fact] + public async Task HeartbeatDuringTransactionOnIdleConnectionDoesNotTearItDown() + { + const int Timeout = 600; + var logs = new CapturingLoggerFactory(); + using var server = new InProcessTestServer(Output); + var config = server.GetClientConfig(defaultOnly: true); + config.SyncTimeout = config.AsyncTimeout = Timeout; + config.HeartbeatInterval = TimeSpan.FromHours(1); // only the heartbeats this test issues + config.KeepAlive = 600; // no pings resetting the last-read clock during the idle period + config.LoggerFactory = logs; + + await using var conn = await ConnectionMultiplexer.ConnectAsync(config); + var db = conn.GetDatabase(1); // a non-default database, so a SELECT prefix is written ahead of the transaction + var key = Me(); + await db.StringSetAsync(key, "before"); + + // the heuristic needs the connection silent for more than 4x the timeout (whole seconds) before the write + await Task.Delay((4 * Timeout) + 1_000); + + // replies now take longer than the gap to the heartbeat, but well inside the timeout + server.SetLatency(TimeSpan.FromMilliseconds(250)); + var tran = db.CreateTransaction(); + tran.AddCondition(Condition.StringEqual(key, "before")); // WATCH + GET, written by the transaction itself + _ = tran.StringSetAsync(key, "after"); + var exec = Task.Run(() => tran.ExecuteAsync()); + + await Task.Delay(50); // the sub-commands are on the wire, their replies are not back yet + conn.OnHeartbeat(); + + var committed = await exec.ContinueWith(t => t.IsCompletedSuccessfully && t.Result, TaskScheduler.Default); + Assert.DoesNotContain(logs.Lines, l => l.Contains("Dead socket", StringComparison.Ordinal)); + Assert.True(committed, "the transaction should have committed"); + Assert.Equal("after", await db.StringGetAsync(key)); + } + + private sealed class CapturingLoggerFactory : ILoggerFactory, ILogger + { + private readonly List _lines = []; + + public IReadOnlyList Lines + { + get { lock (_lines) return [.. _lines]; } + } + + public ILogger CreateLogger(string categoryName) => this; + + public void AddProvider(ILoggerProvider provider) { } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + lock (_lines) _lines.Add(formatter(state, exception)); + } + + public void Dispose() { } + } +} From 9d30a98e545394bb1cd2a01a3ff731dddf691592 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 23 Oct 2026 12:16:07 +0100 Subject: [PATCH 17/22] Refresh package pins: 10.0.13 servicing family and tooling (#3294) Shipped BCL dependencies move 10.0.5 -> 10.0.13 (System.IO.Hashing 10.0.12 -> 10.0.13); test/tooling packages to current. Removes the unreferenced PerformanceCounter pin and documents that the StackExchange.Redis 2.13.17 pin is the baseline projects' previous-major comparison, not a lagging version. Analyzer.Testing 1.1.4 brings NuGet.Packaging 7.0.3 -> System.Security.Cryptography.Pkcs 9.0.6, which NuGet audit rejects; a direct Pkcs 10.0.13 reference in Build.Tests lifts it. Held back for separate PRs: xunit.v3 4.x (needs Microsoft.Testing.Platform 2.5+, which drops the VSTest bridge under the .NET 10 SDK, plus discoverer/attribute changes) and StyleCop 1.3-beta (92 new rule hits in src). --- Directory.Packages.props | 39 ++++++++++--------- .../StackExchange.Redis.Build.Tests.csproj | 1 + 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 1009a1163..ac86b6508 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,16 +1,15 @@ - - - - + + + - - + + - + @@ -27,31 +26,35 @@ 3.3.3 - a version that cannot see descriptors declared outside the analyzer type, as ours are in Diagnostics.cs. Pinned separately because it is a build-time analyzer we never ship: unlike the reference above, its version has nothing to do with the host floor, so upgrade at will. --> - + - + - - + + + + - + - + - + - + + - - + + - + diff --git a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj index 14db431aa..4c85bcf99 100644 --- a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj +++ b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj @@ -24,6 +24,7 @@ + From 9a1fabe1cf377ce20e6d0974d33dd828dfb97225 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 23 Oct 2026 12:31:54 +0100 Subject: [PATCH 18/22] xunit.v3 4.0 on Microsoft.Testing.Platform (#3295) * xunit.v3 4.0 on Microsoft.Testing.Platform xunit.v3 4.x depends on xunit.v3.mtp-v2, which needs Microsoft.Testing.Platform 2.5+, and that drops the VSTest bridge under the .NET 10 SDK. So: global.json selects the MTP runner for dotnet test, the four test projects set UseMicrosoftTestingPlatformRunner and stop referencing the VSTest adapters (Microsoft.NET.Test.Sdk, xunit.runner.visualstudio, xunit.v3.runner.console), and CI.yml/build.ps1 use the MTP dotnet test shape (--project / --solution, --report-xunit-trx; the GitHub Actions reporter self-registers as an extension). xunit 4 fallout: the per-protocol TheoryDiscoverer overrides the new 6-parameter CreateTestCasesForDataRow; the fault-injector tests use [Parallelization(Mode = None)]; xUnit1069 is suppressed for the round-trip tests, whose Timeout is a hang guard; net481 regains System.Reflection.Metadata and Microsoft.CSharp, previously transitive via Test.Sdk. * CI: the Ubuntu composite action also moves to the MTP dotnet test shape; enable the GitHub reporter --report-github is off by default under Microsoft.Testing.Platform (the VSTest logger was on by name), so both jobs ask for it explicitly, with the summary limited to failures as before. * Follow-ups for the MTP runner: analyzer tests join the solution, build.ps1 wording and framework, docs Build.Tests was never in StackExchange.Redis.slnx, so dotnet test --solution skipped it. The -NetCoreOnlyTests path moves to net10.0: two test projects have no net8.0, and the runner fails a project lacking the requested framework instead of skipping it. AGENTS.md, the skills and the fault-injector README use the --project / --filter-class forms. * Use the platform TRX reporter; observe the test-side tasks the new runner reports as unobserved xunit 4.0.2's --report-xunit-trx wrote a truncated file for the 6.8k-test run (ends mid-element; dorny/test-reporter: 'Invalid XML'), locally as well as in CI. Microsoft.Testing.Extensions.TrxReport's --report-trx produces a complete report (validated: parses, 6828 UnitTestResult elements). Microsoft.Testing.Platform also prints every UnobservedTaskException at the end of the run, which VSTest never showed. All of the ones seen are in test code, not the library: the in-process server's background work logging to xunit after its test ended (xunit v3 throws 'no currently active test'); BatchWriteFailureTests never reading its Task.WhenAll's exception; TcpForwarder's losing copy direction. Each now observes or swallows its own failure. * VectorSetInfo: vset-uid is a zero-based counter, so do not assert it is non-zero Measured on a fresh test server: the first vector set created after startup reports vset-uid 0, the next 1, and so on. CI starts a fresh server every run, and the xunit 4 discovery order makes VectorSetInfo(Binary) under RESP2 the first test to create one, so the old assertion failed there and nowhere else. Check the parsed value against the raw VINFO reply instead. --- .../SKILL.md | 4 ++-- .claude/skills/implement-resp-command/SKILL.md | 4 ++-- .github/actions/run-tests/action.yml | 12 ++++++++---- .github/workflows/CI.yml | 5 ++++- AGENTS.md | 12 ++++++++---- Directory.Packages.props | 9 ++++----- StackExchange.Redis.slnx | 1 + build.ps1 | 8 +++++--- global.json | 3 +++ tests/RESPite.Tests/RESPite.Tests.csproj | 7 ++++--- .../StackExchange.Redis.Build.Tests.csproj | 6 ++++-- .../AssemblyInfo.cs | 5 +++-- .../README.md | 2 +- ...tackExchange.Redis.FaultInjector.Tests.csproj | 7 ++++--- .../BatchWriteFailureTests.cs | 3 ++- .../Helpers/Attributes.cs | 4 ++-- .../Helpers/TcpForwarder.cs | 8 +++++--- .../InProcessTestServer.cs | 9 ++++++++- .../StackExchange.Redis.Tests.csproj | 16 +++++++++++++--- .../VectorSetIntegrationTests.cs | 9 +++++++-- 20 files changed, 90 insertions(+), 44 deletions(-) diff --git a/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md b/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md index 4c32691ee..a13cadc9d 100644 --- a/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md +++ b/.claude/skills/create-implementation-plan-for-redis-api-change/SKILL.md @@ -234,8 +234,8 @@ present (write "none" rather than omitting one): malformed), the `RoundTripUnitTests` cases (exact bytes), the live tests written-not-run with `Create(require: RedisFeatures.vX_Y_Z)`, the HLD section 8 scenarios copied and marked `expected`, and the exact commands: - `dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter "FullyQualifiedName~"` - and, when the analyzer changed, `dotnet test tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj`. + `dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-class "**"` + and, when the analyzer changed, `dotnet test --project tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj`. 7. **Docs / changelog / public-API files** - `docs/.md` + `docs/index.md` yes/no, `docs/exp/SERxxx.md` yes/no, the `PublicAPI` files, and the note that `docs/ReleaseNotes.md` is frozen. diff --git a/.claude/skills/implement-resp-command/SKILL.md b/.claude/skills/implement-resp-command/SKILL.md index dff1b62f5..9223d8cad 100644 --- a/.claude/skills/implement-resp-command/SKILL.md +++ b/.claude/skills/implement-resp-command/SKILL.md @@ -145,6 +145,6 @@ The in-process managed server (`toys/StackExchange.Redis.Server`) may also need ## Before finishing - `dotnet build Build.csproj -c Release /p:CI=true` — analyzers + `TreatWarningsAsErrors` must pass (this catches a missing `PublicAPI.Unshipped.txt` entry). -- `dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter "FullyQualifiedName~MyCommand"` — runs your new unit tests without any server. -- `dotnet test tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj` — if you touched `TransactionAnalyzer`. Also needs no server, and takes seconds. +- `dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-class "*MyCommand*"` — runs your new unit tests without any server (Microsoft.Testing.Platform filters take fully qualified names, `*` at either end). +- `dotnet test --project tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj` — if you touched `TransactionAnalyzer`. Also needs no server, and takes seconds. - Double-check no shipped signature changed (back-compat). diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index a6258a3ab..ef016bd6b 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -71,15 +71,19 @@ runs: - name: StackExchange.Redis.Tests shell: bash + # Microsoft.Testing.Platform `dotnet test` (selected in global.json); the GitHub Actions reporter is a platform + # extension that is off by default, and the TRX report feeds dorny/test-reporter below (the platform's + # reporter, not xunit's --report-xunit-trx, which truncated the file in 4.0.2) run: >- - dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj + dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj --no-build -c Release -f net10.0 - --logger trx - --logger "GitHubActions;summary-include-passed=false;summary-include-skipped=false" --results-directory ./test-results/ - /p:CI=true + --report-trx + --report-github + --report-github-summary-include-passed false + --report-github-summary-include-skipped false - uses: dorny/test-reporter@v3 continue-on-error: true diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f4c6a6a6a..1522cf45f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -225,7 +225,10 @@ jobs: run: | $exitCode = 0 foreach ($tfm in @("net10.0", "net481")) { - dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj --no-build -c Release -f $tfm --logger trx --logger GitHubActions --results-directory ./test-results/ /p:CI=true + # Microsoft.Testing.Platform `dotnet test` (selected in global.json); the GitHub Actions reporter is a platform + # extension that is off by default, and the TRX report feeds dorny/test-reporter below (the platform's + # reporter, not xunit's --report-xunit-trx, which truncated the file in 4.0.2) + dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj --no-build -c Release -f $tfm --results-directory ./test-results/ --report-trx --report-github --report-github-summary-include-passed false --report-github-summary-include-skipped false if ($LASTEXITCODE -ne 0) { $exitCode = $LASTEXITCODE } diff --git a/AGENTS.md b/AGENTS.md index 2f07ec7b5..5f484d71e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ StackExchange.Redis — a high-performance .NET client for RESP servers (Redis, - `toys/` — runnable samples and an in-process RESP server (`StackExchange.Redis.Server`, used by tests as a managed fake server), Kestrel-hosted server, console tools. - `docs/` — published documentation site source (markdown). `docs/ReleaseNotes.md` is the changelog (frozen at 3.0 — from v3 onward release notes live in GitHub Releases). -`Build.csproj` is a `Microsoft.Build.Traversal` project that references everything under `eng/src/tests/toys` — build/test/pack it to act on the whole repo. `StackExchange.Redis.slnx` is the IDE solution (XML SLNX format, not classic `.sln`). +`Build.csproj` is a `Microsoft.Build.Traversal` project that references everything under `eng/src/tests/toys` — build/pack it to act on the whole repo. `StackExchange.Redis.slnx` is the IDE solution (XML SLNX format, not classic `.sln`); it is also what `dotnet test --solution` runs, because tests use the Microsoft.Testing.Platform runner (selected in `global.json`), which cannot see a traversal project on the .NET 10 SDK. ## Build, test, pack @@ -30,10 +30,14 @@ pwsh ./build.ps1 -StartServers docker compose --file tests/RedisConfigs/docker-compose.yml up -d --wait # Run the main test suite against one target framework (fastest inner loop) -dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -c Release -f net10.0 +dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -c Release -f net10.0 -# Run a single test class / method (xUnit v3 + Microsoft.Testing.Platform) -dotnet test tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter "FullyQualifiedName~ClassName.MethodName" +# Run a single test class / method (xUnit v3 on Microsoft.Testing.Platform; names are fully qualified, '*' allowed at either end) +dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-class StackExchange.Redis.Tests.ClassName +dotnet test --project tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj -f net10.0 --filter-method StackExchange.Redis.Tests.ClassName.MethodName + +# Every test project at once (what build.ps1 does) +dotnet test --solution StackExchange.Redis.slnx -c Release --no-build # Pack the library dotnet pack src/StackExchange.Redis/StackExchange.Redis.csproj --no-build -c Release /p:Packing=true diff --git a/Directory.Packages.props b/Directory.Packages.props index ac86b6508..2759ef146 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -41,8 +41,9 @@ - - + + + @@ -55,9 +56,7 @@ - - - + \ No newline at end of file diff --git a/StackExchange.Redis.slnx b/StackExchange.Redis.slnx index f3a6ac32f..55d863161 100644 --- a/StackExchange.Redis.slnx +++ b/StackExchange.Redis.slnx @@ -40,6 +40,7 @@ + diff --git a/build.ps1 b/build.ps1 index 3ace75a06..09a2a264c 100644 --- a/build.ps1 +++ b/build.ps1 @@ -29,11 +29,13 @@ if ($RunTests) { & .\RedisConfigs\start-all.cmd Write-Host "Servers Started." -ForegroundColor "Green" } - Write-Host "Running tests: Build.csproj traversal (all frameworks)" -ForegroundColor "Magenta" + Write-Host "Running tests: StackExchange.Redis.slnx (all test projects)" -ForegroundColor "Magenta" if ($NetCoreOnlyTests) { - dotnet test ".\Build.csproj" -c Release -f net8.0 --no-build --logger trx + # net10.0, not net8.0: the analyzer and fault-injector test projects only target net10.0, and the + # Microsoft.Testing.Platform runner fails a project that lacks the requested framework rather than skipping it + dotnet test --solution ".\StackExchange.Redis.slnx" -c Release -f net10.0 --no-build --report-trx } else { - dotnet test ".\Build.csproj" -c Release --no-build --logger trx + dotnet test --solution ".\StackExchange.Redis.slnx" -c Release --no-build --report-trx } if ($LastExitCode -ne 0) { Write-Host "Error with tests, aborting build." -Foreground "Red" diff --git a/global.json b/global.json index 35e954767..e851283da 100644 --- a/global.json +++ b/global.json @@ -2,5 +2,8 @@ { "sdk": { "allowPrerelease": false + }, + "test": { + "runner": "Microsoft.Testing.Platform" } } \ No newline at end of file diff --git a/tests/RESPite.Tests/RESPite.Tests.csproj b/tests/RESPite.Tests/RESPite.Tests.csproj index eb40683a7..74af96a40 100644 --- a/tests/RESPite.Tests/RESPite.Tests.csproj +++ b/tests/RESPite.Tests/RESPite.Tests.csproj @@ -6,14 +6,15 @@ false true Exe + + true - - - + diff --git a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj index 4c85bcf99..060064481 100644 --- a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj +++ b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj @@ -2,6 +2,9 @@ net10.0 Exe + + true enable @@ -25,9 +28,8 @@ - - + diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs index 40188bd5b..170669fc7 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs +++ b/tests/StackExchange.Redis.FaultInjector.Tests/AssemblyInfo.cs @@ -1,4 +1,5 @@ -using Xunit; +using Xunit.Sdk; +using Xunit.v3; // One cluster, one injector, and scenarios that mutate *cluster* state - node exclusions, maintenance mode, // endpoint policies. Running classes in parallel therefore breaks two ways at once: they interfere semantically @@ -8,4 +9,4 @@ // // So this tier is strictly serial. It costs wall-clock - the scenarios are minutes each - and buys results that // mean something. -[assembly: CollectionBehavior(DisableTestParallelization = true)] +[assembly: Parallelization(Mode = ParallelMode.None)] diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/README.md b/tests/StackExchange.Redis.FaultInjector.Tests/README.md index 95c3d3c05..0ae6ff8f2 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/README.md +++ b/tests/StackExchange.Redis.FaultInjector.Tests/README.md @@ -12,7 +12,7 @@ docker compose up -d # 10-15 minutes for the cluster to come export SER_FI_CONFIG_DIR=$PWD # or FI_CONSOLE_CONFIG_DIR, which is also honoured export E2E_SCENARIO_TESTS=true # explicit opt-in: these create and delete databases -dotnet test tests/StackExchange.Redis.FaultInjector.Tests +dotnet test --project tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj ``` **Point it at the *current* environment.** A stale directory passes every check we make: the credentials are diff --git a/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj index 052d85e11..308b3aa29 100644 --- a/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj +++ b/tests/StackExchange.Redis.FaultInjector.Tests/StackExchange.Redis.FaultInjector.Tests.csproj @@ -11,15 +11,16 @@ full enable true + + true - - - + diff --git a/tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs b/tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs index 5ca7a2789..0243b4515 100644 --- a/tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs +++ b/tests/StackExchange.Redis.Tests/BatchWriteFailureTests.cs @@ -46,7 +46,8 @@ public async Task EveryBatchCommandCompletesAfterMidBatchWriteFailure(bool queue { Assert.Fail("hung: " + string.Join(", ", all.Select((t, i) => $"cmd{i}={t.Status}"))); } - // individual faults are inspected below + // individual faults are inspected below; the combined task's own exception is never read, so mark it seen + _ = everything.Exception; for (int i = 0; i < all.Length; i++) { diff --git a/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs b/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs index 275101569..4ef9b633f 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/Attributes.cs @@ -57,8 +57,8 @@ public override ValueTask> Discover(ITestFra public class TheoryDiscoverer : Xunit.v3.TheoryDiscoverer { - protected override ValueTask> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute, ITheoryDataRow dataRow, object?[] testMethodArguments) - => base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments).ExpandAsync(); + protected override ValueTask> CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute, ITheoryDataRow dataRow, object?[] testMethodArguments, string? index) + => base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, testMethodArguments, index).ExpandAsync(); protected override ValueTask> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, IXunitTestMethod testMethod, ITheoryAttribute theoryAttribute) => base.CreateTestCasesForTheory(discoveryOptions, testMethod, theoryAttribute).ExpandAsync(); diff --git a/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs b/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs index ea2b92665..7f8ea83a7 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/TcpForwarder.cs @@ -69,9 +69,11 @@ private async Task PumpAsync(TcpClient client) // the buffer size is spelled out because the two-argument (stream, token) overload does not // exist on .NET Framework, which the test project still targets const int BufferSize = 4096; - await Task.WhenAny( - near.CopyToAsync(far, BufferSize, _cts.Token), - far.CopyToAsync(near, BufferSize, _cts.Token)).ForAwait(); + var inbound = near.CopyToAsync(far, BufferSize, _cts.Token); + var outbound = far.CopyToAsync(near, BufferSize, _cts.Token); + await Task.WhenAny(inbound, outbound).ForAwait(); + // the other direction fails when the sockets go away below; nobody awaits it, so observe it here + _ = Task.WhenAll(inbound, outbound).ContinueWith(static t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously); } catch { diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index 047b6d04d..726eb0bbb 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -181,7 +181,14 @@ public ConfigurationOptions GetClientConfig(bool withPubSub = true, bool default public override void Log(string message) { - _log?.WriteLine(message); + // the server's own background work (write loops, delayed closes after MOVING) can outlive the test that + // created it; xunit v3 throws on output with no active test, and from a fire-and-forget task that only + // surfaces as an unobserved exception at the end of the run + try + { + _log?.WriteLine(message); + } + catch (InvalidOperationException) { } base.Log(message); } diff --git a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj index f20a2646a..a20caaa0f 100644 --- a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj +++ b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj @@ -9,9 +9,17 @@ full enable true + + true $(DefineConstants);BUILD_CURRENT + + + $(NoWarn);xUnit1069 @@ -29,15 +37,17 @@ - + + + + - - + diff --git a/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs index 0515451ba..fb568a1a1 100644 --- a/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs +++ b/tests/StackExchange.Redis.Tests/VectorSetIntegrationTests.cs @@ -252,8 +252,13 @@ public async Task VectorSetInfo(VectorSetQuantization quantization) Assert.Equal(quantization, v.Quantization); Assert.Null(v.QuantizationRaw); // Should be null for known quant types - Assert.NotEqual(0, v.VectorSetUid); - Assert.NotEqual(0, v.HnswMaxNodeUid); + // vset-uid is a zero-based per-process counter: the first vector set created after the server starts + // has uid 0, which is exactly what happens on a fresh CI server when this is the first such test to run. + // So check the parse against the raw reply rather than against zero. + var raw = (await db.ExecuteAsync("VINFO", key)).ToDictionary(); + Assert.Equal((long)raw["vset-uid"], v.VectorSetUid); + Assert.Equal((long)raw["hnsw-max-node-uid"], v.HnswMaxNodeUid); + Assert.NotEqual(0, v.HnswMaxNodeUid); // node uids start at 1, and there is one node } [Fact] From aa1e5c06741361b690abd40733657d6f60fd08ee Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 23 Oct 2026 12:36:04 +0100 Subject: [PATCH 19/22] StyleCop.Analyzers 1.2.0-beta.556 -> 1.3.151-beta, and the tidy-up it asks for (#3296) New rule hits, all fixed rather than suppressed: SA1407 (parenthesise comparisons combined with the non-short-circuit & and |), SA1143 (multi-line conditionals use the expanded layout), SA1504 (an accessor pair is all single-line or all multi-line), SA1315 (override/implementation parameter names match the member they implement), SA1653 ( on secondary partial declarations), plus single SA1117/SA1413/SA1500 sites. 46 sites in RESPite, 175 in StackExchange.Redis, 3 in RESPite.Benchmark. SA1315 found a real bug: KeyPrefixedDatabase.SortedSetRangeAndStore (the sync, key-prefixed overload) declared its first two parameters as (destination, source) against the interface's (source, destination) and forwarded them by name, so it read from the destination and wrote into the source. Regression test: WithKeyPrefixTests.SortedSetRangeAndStoreUsesTheRightKeys. Public API: AzureOptionsProvider/AzureManagedRedisOptionsProvider.AfterConnectAsync rename 'muxer' to the base's 'multiplexer' (recorded in PublicAPI.Unshipped); every other rename is on an internal type or an explicit interface implementation. --- Directory.Packages.props | 2 +- src/RESPite.Benchmark/BenchmarkBase.cs | 12 +-- src/RESPite/Buffers/CycleBuffer.cs | 14 +-- src/RESPite/Internal/BlockBuffer.cs | 8 +- src/RESPite/Messages/RespFrameScanner.cs | 6 +- src/RESPite/Messages/RespReader.Utils.cs | 2 +- src/RESPite/Messages/RespReader.cs | 20 ++-- src/RESPite/Messages/RespScanState.cs | 2 +- src/RESPite/PublicAPI/PublicAPI.Unshipped.txt | 2 + src/RESPite/Shared/AsciiHash.Comparers.cs | 4 +- src/RESPite/Shared/AsciiHash.Instance.cs | 8 +- src/RESPite/Shared/AsciiHash.cs | 6 +- src/StackExchange.Redis/ArrayGrepRequest.cs | 15 ++- .../Availability/CircuitBreaker.cs | 14 +-- .../Availability/FaultContext.cs | 2 +- .../Availability/HealthCheck.Execute.cs | 16 +-- .../Availability/MultiGroupMultiplexer.cs | 12 ++- .../Availability/RetryController.cs | 4 +- src/StackExchange.Redis/BufferReader.cs | 3 +- .../AzureManagedRedisOptionsProvider.cs | 2 +- .../Configuration/AzureOptionsProvider.cs | 4 +- .../Configuration/LoggingTunnel.cs | 6 +- .../ConfigurationOptions.cs | 33 ++++-- .../ConnectionMultiplexer.FeatureFlags.cs | 8 +- .../ConnectionMultiplexer.cs | 18 ++-- src/StackExchange.Redis/CursorEnumerable.cs | 2 +- src/StackExchange.Redis/Format.cs | 12 +-- .../Interfaces/IDatabase.VectorSets.cs | 6 +- .../Interfaces/IDatabaseAsync.VectorSets.cs | 6 +- src/StackExchange.Redis/KeyNotification.cs | 7 +- .../KeyspaceIsolation/KeyPrefixed.cs | 32 +++--- .../KeyspaceIsolation/KeyPrefixedDatabase.cs | 18 ++-- .../Maintenance/MaintenanceFaultSurface.cs | 4 +- src/StackExchange.Redis/MessageWriter.cs | 2 +- src/StackExchange.Redis/PhysicalBridge.cs | 4 +- .../PhysicalConnection.Read.cs | 6 +- .../Profiling/ProfiledCommand.cs | 13 ++- .../PublicAPI/PublicAPI.Unshipped.txt | 4 + src/StackExchange.Redis/RedisChannel.cs | 2 +- src/StackExchange.Redis/RedisDatabase.cs | 101 ++++++++++-------- src/StackExchange.Redis/RedisErrorKind.cs | 2 +- src/StackExchange.Redis/RedisKey.cs | 8 +- src/StackExchange.Redis/RedisResult.cs | 70 +++++++----- src/StackExchange.Redis/RedisServer.cs | 6 +- src/StackExchange.Redis/RedisSubscriber.cs | 2 +- src/StackExchange.Redis/RedisValue.cs | 5 +- src/StackExchange.Redis/RenderedArgs.cs | 3 +- .../RespReaderExtensions.cs | 5 +- src/StackExchange.Redis/ResultBox.cs | 4 +- src/StackExchange.Redis/ResultProcessor.cs | 3 +- src/StackExchange.Redis/ServerEndPoint.cs | 14 ++- .../ServerSelectionStrategy.cs | 3 +- src/StackExchange.Redis/Subscription.cs | 4 +- src/StackExchange.Redis/ValueCondition.cs | 12 +-- .../VectorSetSimilaritySearchMessage.cs | 6 +- .../VectorSetSimilaritySearchRequest.cs | 25 ++++- .../WithKeyPrefixTests.cs | 23 ++++ 57 files changed, 379 insertions(+), 258 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 2759ef146..f968c96c5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -51,7 +51,7 @@ major, so this is not a lagging pin to be bumped to 3.x/4.x when the others are refreshed --> - + diff --git a/src/RESPite.Benchmark/BenchmarkBase.cs b/src/RESPite.Benchmark/BenchmarkBase.cs index 92c399232..6304079f0 100644 --- a/src/RESPite.Benchmark/BenchmarkBase.cs +++ b/src/RESPite.Benchmark/BenchmarkBase.cs @@ -437,19 +437,13 @@ private static string GetNameCore(Delegate underlyingAction, out string descript { string name = underlyingAction.Method.Name; - if (underlyingAction.Method.GetCustomAttribute(typeof(DisplayNameAttribute)) is DisplayNameAttribute - { - DisplayName: { Length: > 0 } - } dna) + if (underlyingAction.Method.GetCustomAttribute(typeof(DisplayNameAttribute)) is DisplayNameAttribute { DisplayName: { Length: > 0 } } dna) { name = dna.DisplayName; } description = ""; - if (underlyingAction.Method.GetCustomAttribute(typeof(DescriptionAttribute)) is DescriptionAttribute - { - Description: { Length: > 0 } - } da) + if (underlyingAction.Method.GetCustomAttribute(typeof(DescriptionAttribute)) is DescriptionAttribute { Description: { Length: > 0 } } da) { description = da.Description; } @@ -568,7 +562,7 @@ private async Task RunAsyncCore( $"{TotalOperations(divisor):###,###,##0} requests completed in {seconds:0.00} seconds, {rate:###,###,##0} ops/sec"); } - if (!Quiet & typeof(T) != typeof(DBNull)) + if (!Quiet & (typeof(T) != typeof(DBNull))) { const string format = "Typical result: {0}"; diff --git a/src/RESPite/Buffers/CycleBuffer.cs b/src/RESPite/Buffers/CycleBuffer.cs index c674e6bec..adc15c60e 100644 --- a/src/RESPite/Buffers/CycleBuffer.cs +++ b/src/RESPite/Buffers/CycleBuffer.cs @@ -105,7 +105,7 @@ private void CopyDueToDiscardDuringWrite(int count) .CopyTo(full.Slice(targetOffset, count)); } } - public bool CommittedIsEmpty => ReferenceEquals(startSegment, endSegment) & endSegmentCommitted == 0; + public bool CommittedIsEmpty => ReferenceEquals(startSegment, endSegment) & (endSegmentCommitted == 0); /// /// Marks committed data as fully consumed; it will no longer appear in later calls to . @@ -117,8 +117,8 @@ public void DiscardCommitted(int count) // optimize for most common case, where we consume everything if (ReferenceEquals(startSegment, endSegment) - & count == endSegmentCommitted - & count > 0) + & (count == endSegmentCommitted) + & (count > 0)) { /* we are consuming all the data in the single segment; we can @@ -144,8 +144,8 @@ public void DiscardCommitted(long count) // optimize for most common case, where we consume everything if (ReferenceEquals(startSegment, endSegment) - & count == endSegmentCommitted - & count > 0) // checks sign *and* non-trimmed + & (count == endSegmentCommitted) + & (count > 0)) // checks sign *and* non-trimmed { // see for logic endSegmentCommitted = 0; @@ -339,7 +339,7 @@ public bool TryGetFirstCommittedMemory(int minBytes, out ReadOnlyMemory me // partial segment (and we know it isn't empty) memory = memory.Slice(start: 0, length: available); - return available >= minBytes & minBytes > 0; // last check here applies the -ve logic + return (available >= minBytes) & (minBytes > 0); // last check here applies the -ve logic } // multi-page; hand out the first page (which is, by definition: full) @@ -603,7 +603,7 @@ public int Untrim(bool expandBackwards = false) { _flags &= ~(Flags.StartTrim | Flags.EndTrim); Memory = fullMemory; - if (expandBackwards & RunningIndex >= delta) + if (expandBackwards & (RunningIndex >= delta)) { // push our origin earlier; only valid if // we're the first segment, otherwise diff --git a/src/RESPite/Internal/BlockBuffer.cs b/src/RESPite/Internal/BlockBuffer.cs index 752d74c8d..efdcfaa77 100644 --- a/src/RESPite/Internal/BlockBuffer.cs +++ b/src/RESPite/Internal/BlockBuffer.cs @@ -114,8 +114,8 @@ public static BlockBuffer GetBuffer(BlockBufferSerializer parent, int sizeHint) private int AvailableWithResetIfUseful() { if (_finalizedOffset != 0 // at least some chunks have been finalized - && Volatile.Read(ref _refCount) == 1 // all finalized chunks returned - & _writeOffset == _finalizedOffset) // we're not in the middle of serializing something new + && (Volatile.Read(ref _refCount) == 1) // all finalized chunks returned + & (_writeOffset == _finalizedOffset)) // we're not in the middle of serializing something new { _writeOffset = _finalizedOffset = 0; // swipe left } @@ -207,8 +207,8 @@ private void CopyFrom(Span source) private bool TryResizeFor(int extraBytes) { - if (_finalizedOffset == 0 & // we can only do this if there are no other messages in the buffer - Volatile.Read(ref _refCount) == 1) // and no-one else is looking (we already tried reset) + if ((_finalizedOffset == 0) & // we can only do this if there are no other messages in the buffer + (Volatile.Read(ref _refCount) == 1)) // and no-one else is looking (we already tried reset) { // we're already on the boundary - don't scrimp; just do the math from the end of the buffer byte[] newArray = _arrayPool.Rent(_array.Length + extraBytes); diff --git a/src/RESPite/Messages/RespFrameScanner.cs b/src/RESPite/Messages/RespFrameScanner.cs index 28cd92de7..ac974aee5 100644 --- a/src/RESPite/Messages/RespFrameScanner.cs +++ b/src/RESPite/Messages/RespFrameScanner.cs @@ -98,7 +98,7 @@ private static OperationStatus TryFastRead(ReadOnlySpan data, ref RespScan lo = ((uint)UnsafeCpuUInt16(data, 4)) << 16; break; case 7: - lo = ((uint)UnsafeCpuUInt16(data, 4)) << 16 | ((uint)UnsafeCpuByte(data, 6)) << 8; + lo = (((uint)UnsafeCpuUInt16(data, 4)) << 16) | (((uint)UnsafeCpuByte(data, 6)) << 8); break; default: lo = UnsafeCpuUInt32(data, 4); @@ -123,7 +123,7 @@ private static OperationStatus TryFastRead(ReadOnlySpan data, ref RespScan /// public OperationStatus TryRead(ref RespScanState state, in ReadOnlySequence data) { - if (!_pubsub & state.TotalBytes == 0 & data.IsSingleSegment) + if (!_pubsub & (state.TotalBytes == 0) & data.IsSingleSegment) { var status = TryFastRead(data.FirstSpan, ref state); if (status != UseReader) return status; @@ -148,7 +148,7 @@ static OperationStatus TryReadViaReader(ref RespScanState state, in ReadOnlySequ /// public OperationStatus TryRead(ref RespScanState state, ReadOnlySpan data) { - if (!_pubsub & state.TotalBytes == 0) + if (!_pubsub & (state.TotalBytes == 0)) { #if NET var status = TryFastRead(data, ref state); diff --git a/src/RESPite/Messages/RespReader.Utils.cs b/src/RESPite/Messages/RespReader.Utils.cs index 4a504ec1f..076b3ba28 100644 --- a/src/RESPite/Messages/RespReader.Utils.cs +++ b/src/RESPite/Messages/RespReader.Utils.cs @@ -349,7 +349,7 @@ public readonly byte[] Serialize() target = target.Slice(span.Length); if (!reader.TryMoveToNextSegment()) break; } - if (remaining != 0 | !target.IsEmpty) ThrowEof(); + if ((remaining != 0) | !target.IsEmpty) ThrowEof(); return arr; } } diff --git a/src/RESPite/Messages/RespReader.cs b/src/RESPite/Messages/RespReader.cs index 80ab55a46..c532da143 100644 --- a/src/RESPite/Messages/RespReader.cs +++ b/src/RESPite/Messages/RespReader.cs @@ -240,7 +240,8 @@ private readonly bool ScalarLengthIsSlow(int expected) /// the API, using the API to update the outer reader. public readonly int AggregateLength() => (_flags & (RespFlags.IsAggregate | RespFlags.IsStreaming)) == RespFlags.IsAggregate - ? _length : AggregateLengthSlow(); + ? _length + : AggregateLengthSlow(); /// /// Indicates whether the number of child elements associated with an aggregate is exactly the specified value. @@ -250,7 +251,8 @@ public readonly int AggregateLength() => /// i.e. a map of the form %2\r\n... will report 4 as the length. public readonly bool AggregateLengthIs(int count) => (_flags & (RespFlags.IsAggregate | RespFlags.IsStreaming)) == RespFlags.IsAggregate - ? _length == count : AggregateLengthIsSlow(count); + ? _length == count + : AggregateLengthIsSlow(count); public delegate T Projection(ref RespReader value); @@ -385,7 +387,7 @@ internal readonly bool IsNonNullAggregate internal int Delta() => (_flags & (RespFlags.IsScalar | RespFlags.IsAggregate | RespFlags.IsStreaming | RespFlags.IsAttribute)) switch { - RespFlags.IsScalar | RespFlags.IsAggregate=> -1, // null has this + RespFlags.IsScalar | RespFlags.IsAggregate => -1, // null has this RespFlags.IsScalar => -1, RespFlags.IsAggregate => _length - 1, RespFlags.IsAggregate | RespFlags.IsAttribute => _length, @@ -423,7 +425,7 @@ private bool TryReadNextSkipAttributes(bool skipStreamTerminator) { SkipChildren(); } - else if (skipStreamTerminator & Prefix is RespPrefix.StreamTerminator) + else if (skipStreamTerminator & (Prefix is RespPrefix.StreamTerminator)) { // skip terminator } @@ -446,7 +448,7 @@ private bool TryReadNextProcessAttributes(RespAttributeReader respAttribut { respAttributeReader.Read(ref this, ref attributes); } - else if (skipStreamTerminator & Prefix is RespPrefix.StreamTerminator) + else if (skipStreamTerminator & (Prefix is RespPrefix.StreamTerminator)) { // skip terminator } @@ -1657,7 +1659,7 @@ internal readonly bool IsOK() // go mad with this, because it is used so often if (TryGetSpan(out var span) && span.Length == 2) { var u16 = Unsafe.ReadUnaligned(ref UnsafeCurrent); - return u16 == RespConstants.OKUInt16 | u16 == RespConstants.OKUInt16_LC; + return (u16 == RespConstants.OKUInt16) | (u16 == RespConstants.OKUInt16_LC); } return IsSlow(RespConstants.OKBytes, RespConstants.OKBytes_LC); @@ -1893,7 +1895,7 @@ public readonly bool TryReadInt64(out long value) var span = Buffer(stackalloc byte[RespConstants.MaxRawBytesInt64 + 1]); if (span.Length <= RespConstants.MaxRawBytesInt64) { - return Utf8Parser.TryParse(span, out value, out int bytes) & bytes == span.Length; + return Utf8Parser.TryParse(span, out value, out int bytes) & (bytes == span.Length); } value = 0; @@ -1927,7 +1929,7 @@ public readonly bool TryReadInt32(out int value) var span = Buffer(stackalloc byte[RespConstants.MaxRawBytesInt32 + 1]); if (span.Length <= RespConstants.MaxRawBytesInt32) { - return Utf8Parser.TryParse(span, out value, out int bytes) & bytes == span.Length; + return Utf8Parser.TryParse(span, out value, out int bytes) & (bytes == span.Length); } value = 0; @@ -2015,7 +2017,7 @@ internal readonly bool UnsafeTryReadShortAscii(out string value, int maxLength = foreach (var b in span) { // allow [SPACE]-thru-[DEL], plus CR/LF - if (!(b < 127 & (b >= 32 | (b is 12 or 13)))) + if (!((b < 127) & ((b >= 32) | (b is 12 or 13)))) { return false; } diff --git a/src/RESPite/Messages/RespScanState.cs b/src/RESPite/Messages/RespScanState.cs index 37cd3f8b6..ef5588ee6 100644 --- a/src/RESPite/Messages/RespScanState.cs +++ b/src/RESPite/Messages/RespScanState.cs @@ -127,7 +127,7 @@ private long ReadCore(ref RespReader reader, long startOffset = 0) #if DEBUG _elementCount++; #endif - if (!reader.IsAttribute & _prefix == RespPrefix.None) + if (!reader.IsAttribute & (_prefix == RespPrefix.None)) { _prefix = reader.Prefix; } diff --git a/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt b/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt index 7dc5c5811..555737f80 100644 --- a/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/RESPite/PublicAPI/PublicAPI.Unshipped.txt @@ -1 +1,3 @@ #nullable enable +*REMOVED*[SER004]override RESPite.AsciiHash.Equals(object? other) -> bool +[SER004]override RESPite.AsciiHash.Equals(object? obj) -> bool diff --git a/src/RESPite/Shared/AsciiHash.Comparers.cs b/src/RESPite/Shared/AsciiHash.Comparers.cs index 7b69a15a4..bc3bd5ff7 100644 --- a/src/RESPite/Shared/AsciiHash.Comparers.cs +++ b/src/RESPite/Shared/AsciiHash.Comparers.cs @@ -13,7 +13,7 @@ private CaseSensitiveComparer() { } public bool Equals(AsciiHash x, AsciiHash y) { var len = x.Length; - return (len == y.Length & x._hashCS == y._hashCS) + return ((len == y.Length) & (x._hashCS == y._hashCS)) && (len <= MaxBytesHashed || x.Span.SequenceEqual(y.Span)); } @@ -28,7 +28,7 @@ private CaseInsensitiveComparer() { } public bool Equals(AsciiHash x, AsciiHash y) { var len = x.Length; - return (len == y.Length & x._hashUC == y._hashUC) + return ((len == y.Length) & (x._hashUC == y._hashUC)) && (len <= MaxBytesHashed || SequenceEqualsCI(x.Span, y.Span)); } diff --git a/src/RESPite/Shared/AsciiHash.Instance.cs b/src/RESPite/Shared/AsciiHash.Instance.cs index 93b8258ca..94a774f81 100644 --- a/src/RESPite/Shared/AsciiHash.Instance.cs +++ b/src/RESPite/Shared/AsciiHash.Instance.cs @@ -37,13 +37,13 @@ public AsciiHash(string? value) : this(value is null ? [] : Encoding.ASCII.GetBy public override string ToString() => _length == 0 ? "" : Encoding.ASCII.GetString(_arr, _index, _length); /// - public override bool Equals(object? other) => other is AsciiHash hash && Equals(hash); + public override bool Equals(object? obj) => obj is AsciiHash hash && Equals(hash); /// [CLSCompliant(false)] public bool Equals(in AsciiHash other) { - return (_length == other.Length & _hashCS == other._hashCS) + return ((_length == other.Length) & (_hashCS == other._hashCS)) && (_length <= MaxBytesHashed || Span.SequenceEqual(other.Span)); } @@ -65,7 +65,7 @@ public bool IsCS(ReadOnlySpan value) { var cs = HashCS(value); var len = _length; - if (cs != _hashCS | value.Length != len) return false; + if ((cs != _hashCS) | (value.Length != len)) return false; return len <= MaxBytesHashed || Span.SequenceEqual(value); } @@ -73,7 +73,7 @@ public bool IsCI(ReadOnlySpan value) { var uc = HashUC(value); var len = _length; - if (uc != _hashUC | value.Length != len) return false; + if ((uc != _hashUC) | (value.Length != len)) return false; return len <= MaxBytesHashed || SequenceEqualsCI(Span, value); } diff --git a/src/RESPite/Shared/AsciiHash.cs b/src/RESPite/Shared/AsciiHash.cs index 15164c4fa..3fa732343 100644 --- a/src/RESPite/Shared/AsciiHash.cs +++ b/src/RESPite/Shared/AsciiHash.cs @@ -106,7 +106,7 @@ public static unsafe bool SequenceEqualsCI(ReadOnlySpan first, ReadOnlySpa { byte x = firstPtr[i]; var xCI = x & CS_MASK; - if (xCI >= 'A' & xCI <= 'Z') + if ((xCI >= 'A') & (xCI <= 'Z')) { // alpha mismatch if (xCI != (secondPtr[i] & CS_MASK)) return false; @@ -171,7 +171,7 @@ public static unsafe bool SequenceEqualsCI(ReadOnlySpan first, ReadOnlySpa { int x = (byte)firstPtr[i]; var xCI = x & CS_MASK; - if (xCI >= 'A' & xCI <= 'Z') + if ((xCI >= 'A') & (xCI <= 'Z')) { // alpha mismatch if (xCI != (secondPtr[i] & CS_MASK)) return false; @@ -206,7 +206,7 @@ public static unsafe bool SequenceEqualsCI(ReadOnlySpan first, ReadOnlySpa { int x = (byte)firstPtr[i]; var xCI = x & CS_MASK; - if (xCI >= 'A' & xCI <= 'Z') + if ((xCI >= 'A') & (xCI <= 'Z')) { // alpha mismatch if (xCI != (secondPtr[i] & CS_MASK)) return false; diff --git a/src/StackExchange.Redis/ArrayGrepRequest.cs b/src/StackExchange.Redis/ArrayGrepRequest.cs index bf9e34f8f..7f00cf147 100644 --- a/src/StackExchange.Redis/ArrayGrepRequest.cs +++ b/src/StackExchange.Redis/ArrayGrepRequest.cs @@ -59,7 +59,10 @@ private void SetFlag(LocalFlags flag, bool value) /// public RedisArrayIndex? Start { - get => GetFlag(LocalFlags.StartSpecified) ? _start : null; + get + { + return GetFlag(LocalFlags.StartSpecified) ? _start : null; + } set { if (value.HasValue) @@ -84,7 +87,10 @@ public RedisArrayIndex? Start /// public RedisArrayIndex? End { - get => GetFlag(LocalFlags.EndSpecified) ? _end : null; + get + { + return GetFlag(LocalFlags.EndSpecified) ? _end : null; + } set { if (value.HasValue) @@ -110,7 +116,10 @@ public RedisArrayIndex? End /// Corresponds to the LIMIT parameter. public long? Limit { - get => GetFlag(LocalFlags.LimitSpecified) ? _limit : null; + get + { + return GetFlag(LocalFlags.LimitSpecified) ? _limit : null; + } set { if (value.HasValue) diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs index ff232cf7b..4b91a6675 100644 --- a/src/StackExchange.Redis/Availability/CircuitBreaker.cs +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -77,11 +77,11 @@ public CircuitBreaker Create() if (MetricsWindowSize <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "A positive window is required."); if (MetricsWindowSize.TotalSeconds > int.MaxValue) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "The window is too large."); - if ((FailureRateThreshold is DefaultFailureRateThreshold + if (((FailureRateThreshold is DefaultFailureRateThreshold) #if NET8_0_OR_GREATER - & TimeProvider is null + & (TimeProvider is null) #endif - & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures) + & (MinimumNumberOfFailures is DefaultMinimumNumberOfFailures)) && MetricsWindowSize == DefaultMetricsWindowSize) return DefaultInstance; @@ -181,7 +181,7 @@ private sealed class NulAccumulator : Accumulator { public static readonly NulAccumulator AccumulatorInstance = new(); private NulAccumulator() { } - public override void ObserveResult(in FaultContext context) { } + public override void ObserveResult(in FaultContext fault) { } public override bool IsHealthy() => true; public override void Reset() { } } @@ -298,7 +298,7 @@ public void Reset() } } - public override void ObserveResult(in FaultContext result) + public override void ObserveResult(in FaultContext fault) { // which time-slice are we in, and where does it live in the ring? long epoch = breaker.GetEpoch(); @@ -311,7 +311,7 @@ public override void ObserveResult(in FaultContext result) // only getting results one at a time, so we shouldn't be over-stomping much *anyway* Span buckets = _buckets; // *not* a payload copy; this is in-place over the data ref Bucket bucket = ref buckets[index]; - bucket.Count(epoch, success: !result.IsFault); + bucket.Count(epoch, success: !fault.IsFault); } public override bool IsHealthy() => Evaluate(breaker.GetEpoch()); @@ -336,7 +336,7 @@ private bool Evaluate(long epoch) } // don't act until we've seen enough failures to be statistically meaningful - if (total is 0 | failures < breaker._minimumNumberOfFailures) + if ((total is 0) | (failures < breaker._minimumNumberOfFailures)) { return true; } diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs index 3c43f10f8..7ec643082 100644 --- a/src/StackExchange.Redis/Availability/FaultContext.cs +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -53,7 +53,7 @@ public FaultContext(Exception fault) NotApplied = IsKnownNotApplied(kind, status); - if (kind is not RedisErrorKind.None & _connectionFailureType is ConnectionFailureType.None) + if ((kind is not RedisErrorKind.None) & (_connectionFailureType is ConnectionFailureType.None)) { // fill in some blanks switch (kind) diff --git a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs index 9869d63a9..13ca8694d 100644 --- a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs +++ b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs @@ -11,9 +11,11 @@ public sealed partial class HealthCheck /// Evaluate the health of the specified multiplexer, by evaluating all endpoints. /// public Task CheckHealthAsync(IConnectionMultiplexer multiplexer) - => !IsEnabled ? HealthCheckProbe.InconclusiveTask - : multiplexer.IsConnected ? CheckHealthCoreAsync(multiplexer) - : HealthCheckProbe.UnhealthyTask; + => !IsEnabled + ? HealthCheckProbe.InconclusiveTask + : multiplexer.IsConnected + ? CheckHealthCoreAsync(multiplexer) + : HealthCheckProbe.UnhealthyTask; private async Task CheckHealthCoreAsync(IConnectionMultiplexer multiplexer) { @@ -171,9 +173,11 @@ internal static void PutReusablePending(ref Task[]? field, re /// Evaluate the health of an endpoint. /// public Task CheckHealthAsync(IServer server) - => !IsEnabled ? HealthCheckProbe.InconclusiveTask - : server.IsConnected ? CheckHealthCoreAsync(server) - : HealthCheckProbe.UnhealthyTask; + => !IsEnabled + ? HealthCheckProbe.InconclusiveTask + : server.IsConnected + ? CheckHealthCoreAsync(server) + : HealthCheckProbe.UnhealthyTask; private async Task CheckHealthCoreAsync(IServer server) { diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs index 82b8ebc4d..987f3ea93 100644 --- a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -367,9 +367,11 @@ internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheckResul var oldConnected = IsConnected; IsConnected = isConnected; - return isConnected == oldConnected ? GroupConnectionChangedEventArgs.ChangeType.Unknown - : isConnected ? GroupConnectionChangedEventArgs.ChangeType.Reconnected - : GroupConnectionChangedEventArgs.ChangeType.Disconnected; + return isConnected == oldConnected + ? GroupConnectionChangedEventArgs.ChangeType.Unknown + : isConnected + ? GroupConnectionChangedEventArgs.ChangeType.Reconnected + : GroupConnectionChangedEventArgs.ChangeType.Disconnected; } internal void UpdateLatency() @@ -643,7 +645,7 @@ static bool TryGetHealthCheck(object? target, out TimeSpan interval) // note the interval is a group-level concern (how often we re-evaluate the active // member), not a property of any individual health-check interval = typed._options.HealthCheckInterval; - return interval > TimeSpan.Zero & interval != TimeSpan.MaxValue; + return (interval > TimeSpan.Zero) & (interval != TimeSpan.MaxValue); } interval = TimeSpan.Zero; @@ -1249,7 +1251,7 @@ public ISubscriber GetSubscriber(object? asyncState = null) public IDatabase GetDatabase(int db = -1, object? asyncState = null) { - if (asyncState is null & db >= -1 & db <= ConnectionMultiplexer.MaxCachedDatabaseInstance) + if ((asyncState is null) & (db >= -1) & (db <= ConnectionMultiplexer.MaxCachedDatabaseInstance)) { return _databases[db + 1] ??= new MultiGroupDatabase(this, db, null); } diff --git a/src/StackExchange.Redis/Availability/RetryController.cs b/src/StackExchange.Redis/Availability/RetryController.cs index a47d79bbc..f055d0b4e 100644 --- a/src/StackExchange.Redis/Availability/RetryController.cs +++ b/src/StackExchange.Redis/Availability/RetryController.cs @@ -62,7 +62,7 @@ public Task WatchConflictDelayAsync() /// Whether it is ever worth capturing the next-failover token: only when there is more than one /// attempt and the failover threshold sits below the attempt cap. /// - public bool TracksFailover => _maxAttempts > 1 & _maxBeforeFailover < _maxAttempts; + public bool TracksFailover => (_maxAttempts > 1) & (_maxBeforeFailover < _maxAttempts); public bool CanRetry( int attempt, @@ -100,7 +100,7 @@ public bool CanRetry( // by count, we should really switch over to the failover now; is failover available *and* are we allowed? delay = failover; failover = CancellationToken.None; // only failover once - return delay.CanBeCanceled & (policy & RetryResult.FailoverServer) != 0; + return delay.CanBeCanceled & ((policy & RetryResult.FailoverServer) != 0); } // can we pause and retry on the same server? diff --git a/src/StackExchange.Redis/BufferReader.cs b/src/StackExchange.Redis/BufferReader.cs index 22b36ccb6..8b4d2b4b5 100644 --- a/src/StackExchange.Redis/BufferReader.cs +++ b/src/StackExchange.Redis/BufferReader.cs @@ -78,7 +78,8 @@ public ConsumeResult TryConsumeCRLF() default: var offset = OffsetThisSpan; var result = _current[offset++] == (byte)'\r' && _current[offset] == (byte)'\n' - ? ConsumeResult.Success : ConsumeResult.Failure; + ? ConsumeResult.Success + : ConsumeResult.Failure; Consume(2); return result; } diff --git a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs index 1ab612bea..31177de71 100644 --- a/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureManagedRedisOptionsProvider.cs @@ -59,7 +59,7 @@ private bool IsHostInDomains(string hostName, string[] domains) } /// - public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action log) + public override Task AfterConnectAsync(ConnectionMultiplexer multiplexer, Action log) => Task.CompletedTask; /// diff --git a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs index ee577eb59..0f9810f1c 100644 --- a/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs +++ b/src/StackExchange.Redis/Configuration/AzureOptionsProvider.cs @@ -61,8 +61,8 @@ private bool IsHostInDomains(string hostName, string[] domains) } /// - public override Task AfterConnectAsync(ConnectionMultiplexer muxer, Action log) - => AzureMaintenanceEvent.AddListenerAsync(muxer, log); + public override Task AfterConnectAsync(ConnectionMultiplexer multiplexer, Action log) + => AzureMaintenanceEvent.AddListenerAsync(multiplexer, log); /// public override bool GetDefaultSsl(EndPointCollection endPoints) diff --git a/src/StackExchange.Redis/Configuration/LoggingTunnel.cs b/src/StackExchange.Redis/Configuration/LoggingTunnel.cs index 3fb023c9f..5321c49a0 100644 --- a/src/StackExchange.Redis/Configuration/LoggingTunnel.cs +++ b/src/StackExchange.Redis/Configuration/LoggingTunnel.cs @@ -391,14 +391,16 @@ private void CreateLogFiles(EndPoint endpoint, ConnectionType connectionType, ou /// public override ValueTask BeforeSocketConnectAsync(EndPoint endPoint, ConnectionType connectionType, Socket? socket, CancellationToken cancellationToken) { - return _tail is null ? base.BeforeSocketConnectAsync(endPoint, connectionType, socket, cancellationToken) + return _tail is null + ? base.BeforeSocketConnectAsync(endPoint, connectionType, socket, cancellationToken) : _tail.BeforeSocketConnectAsync(endPoint, connectionType, socket, cancellationToken); } /// public override ValueTask GetSocketConnectEndpointAsync(EndPoint endpoint, CancellationToken cancellationToken) { - return _tail is null ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) + return _tail is null + ? base.GetSocketConnectEndpointAsync(endpoint, cancellationToken) : _tail.GetSocketConnectEndpointAsync(endpoint, cancellationToken); } diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 90e12c560..45e2404d3 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -406,7 +406,10 @@ private void SetWithValue(OptionFlags hasValue, ref T storage, T value) where /// public DefaultOptionsProvider Defaults { - get => defaultOptions ??= DefaultOptionsProvider.GetProvider(EndPoints); + get + { + return defaultOptions ??= DefaultOptionsProvider.GetProvider(EndPoints); + } set { defaultOptions = value; @@ -710,7 +713,10 @@ public CommandMap CommandMap Proxy.Envoyproxy => CommandMap.Envoyproxy, _ => CommandMap.Default, }; - set => commandMap = value ?? throw new ArgumentNullException(nameof(value)); + set + { + commandMap = value ?? throw new ArgumentNullException(nameof(value)); + } } /// @@ -746,7 +752,10 @@ public int ConnectTimeout var defaultTimeout = Defaults.ConnectTimeout; return defaultTimeout.HasValue ? (int)defaultTimeout.GetValueOrDefault().TotalMilliseconds : Math.Max(5000, SyncTimeout); } - set => SetWithValue(OptionFlags.ConnectTimeoutHasValue, ref connectTimeout, value); + set + { + SetWithValue(OptionFlags.ConnectTimeoutHasValue, ref connectTimeout, value); + } } /// @@ -1306,7 +1315,8 @@ public string ToString(bool includePassword) commandMap?.AppendDeltas(sb); return sb.ToString(); - static string FormatProtocol(RedisProtocol protocol) => protocol switch { + static string FormatProtocol(RedisProtocol protocol) => protocol switch + { RedisProtocol.Resp2 => "resp2", RedisProtocol.Resp3 => "resp3", _ => protocol.ToString(), @@ -1686,7 +1696,10 @@ public MaintenanceEndpointType MaintenanceMovingEndpointType get => HasValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue) ? _maintenanceMovingEndpointType : Defaults.MaintenanceMovingEndpointType; - set => SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, value); + set + { + SetWithValue(OptionFlags.MaintenanceMovingEndpointTypeHasValue, ref _maintenanceMovingEndpointType, value); + } } /// @@ -1720,7 +1733,10 @@ public TimeSpan MaintenanceRelaxedWindowMax get => HasValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue) ? _maintenanceRelaxedWindowMax : Defaults.MaintenanceRelaxedWindowMax ?? Multiply(MaintenanceRelaxedTimeout, 3); - set => SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, value); + set + { + SetWithValue(OptionFlags.MaintenanceRelaxedWindowMaxHasValue, ref _maintenanceRelaxedWindowMax, value); + } } /// @@ -1739,7 +1755,10 @@ public TimeSpan MaintenancePostEventRelaxedDuration get => HasValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue) ? _maintenancePostEventRelaxedDuration : Defaults.MaintenancePostEventRelaxedDuration ?? Multiply(MaintenanceRelaxedTimeout, 2); - set => SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, value); + set + { + SetWithValue(OptionFlags.MaintenancePostEventRelaxedDurationHasValue, ref _maintenancePostEventRelaxedDuration, value); + } } // TimeSpan * int is netstandard2.1+, so this is ticks arithmetic to keep the down-level TFMs building diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs b/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs index b22d709f3..41272197d 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs @@ -84,12 +84,12 @@ public static bool GetFeatureFlag(string flag) /// there is no separate subscription connection, so asking about answers /// about the shared one. /// - bool? IInternalConnectionMultiplexer.IsSyncReader(EndPoint endpoint, ConnectionType connectionType) - => GetPhysical(endpoint, connectionType)?.IsSyncReader; + bool? IInternalConnectionMultiplexer.IsSyncReader(EndPoint endPoint, ConnectionType type) + => GetPhysical(endPoint, type)?.IsSyncReader; /// As IsSyncReader, for the writer. - bool? IInternalConnectionMultiplexer.IsSyncWriter(EndPoint endpoint, ConnectionType connectionType) - => GetPhysical(endpoint, connectionType)?.IsSyncWriter; + bool? IInternalConnectionMultiplexer.IsSyncWriter(EndPoint endPoint, ConnectionType type) + => GetPhysical(endPoint, type)?.IsSyncWriter; private PhysicalConnection? GetPhysical(EndPoint endpoint, ConnectionType connectionType) => TryResolveServerEndPoint(endpoint)?.GetBridge(connectionType, create: false)?.Physical; diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index 14b2815b0..50c57162a 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -95,8 +95,8 @@ public sealed partial class ConnectionMultiplexer : IInternalConnectionMultiplex private int _activeHeartbeatErrors, lastHeartbeatTicks; internal long LastHeartbeatSecondsAgo => pulse is null - ? -1 - : unchecked(Environment.TickCount - Volatile.Read(ref lastHeartbeatTicks)) / 1000; + ? -1 + : unchecked(Environment.TickCount - Volatile.Read(ref lastHeartbeatTicks)) / 1000; private static int lastGlobalHeartbeatTicks = Environment.TickCount; internal static long LastGlobalHeartbeatSecondsAgo => @@ -1003,7 +1003,8 @@ ServerEndPoint IInternalConnectionMultiplexer.GetServerEndPoint(EndPoint endpoin // a retired server may still be referenced by an alias for a moment; never hand one back, or the // caller receives something whose bridges are gone return _serverIdentities.TryGetValue(endpoint, out var byIdentity) && !byIdentity.IsDisposed - ? byIdentity : null; + ? byIdentity + : null; } // bumped once per applied cluster topology; absence is measured in these rather than in time, so a @@ -1483,7 +1484,8 @@ public IDatabase GetDatabase(int db = -1, object? asyncState = null) // if there's no async-state, and the DB is suitable, we can hand out a re-used instance return (asyncState == null && db <= MaxCachedDatabaseInstance) - ? GetCachedDatabaseInstance(db) : new RedisDatabase(this, db, asyncState); + ? GetCachedDatabaseInstance(db) + : new RedisDatabase(this, db, asyncState); } // DB zero is stored separately, since 0-only is a massively common use-case @@ -1565,8 +1567,8 @@ public IServer GetServer(string host, int port, object? asyncState = null) => /// The async state to pass into the resulting . public IServer GetServer(string hostAndPort, object? asyncState = null) => Format.TryParseEndPoint(hostAndPort, out var ep) - ? GetServer(ep, asyncState) - : throw new ArgumentException($"The specified host and port could not be parsed: {hostAndPort}", nameof(hostAndPort)); + ? GetServer(ep, asyncState) + : throw new ArgumentException($"The specified host and port could not be parsed: {hostAndPort}", nameof(hostAndPort)); /// /// Obtain a configuration API for an individual server. @@ -3020,8 +3022,8 @@ private Task[] QuitAllServers() return quits; } - long? IInternalConnectionMultiplexer.GetConnectionId(EndPoint endpoint, ConnectionType type) - => TryResolveServerEndPoint(endpoint)?.GetBridge(type)?.ConnectionId; + long? IInternalConnectionMultiplexer.GetConnectionId(EndPoint endPoint, ConnectionType type) + => TryResolveServerEndPoint(endPoint)?.GetBridge(type)?.ConnectionId; internal uint UpdateLatency() { diff --git a/src/StackExchange.Redis/CursorEnumerable.cs b/src/StackExchange.Redis/CursorEnumerable.cs index 670ab178d..5f05e12ba 100644 --- a/src/StackExchange.Redis/CursorEnumerable.cs +++ b/src/StackExchange.Redis/CursorEnumerable.cs @@ -108,7 +108,7 @@ public T Current [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - Debug.Assert(_pageOffset >= 0 & _pageOffset < _pageCount & _pageOversized!.Length >= _pageCount); + Debug.Assert((_pageOffset >= 0) & (_pageOffset < _pageCount) & (_pageOversized!.Length >= _pageCount)); return _pageOversized[_pageOffset]; } } diff --git a/src/StackExchange.Redis/Format.cs b/src/StackExchange.Redis/Format.cs index 8907182df..0c4e33272 100644 --- a/src/StackExchange.Redis/Format.cs +++ b/src/StackExchange.Redis/Format.cs @@ -233,10 +233,10 @@ internal static bool TryParseUInt64(string s, out ulong value) => ulong.TryParse(s, NumberStyles.Integer, NumberFormatInfo.InvariantInfo, out value); internal static bool TryParseUInt64(ReadOnlySpan s, out ulong value) => - Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & bytes == s.Length; + Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & (bytes == s.Length); internal static bool TryParseInt64(ReadOnlySpan s, out long value) => - Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & bytes == s.Length; + Utf8Parser.TryParse(s, out value, out int bytes, standardFormat: 'D') & (bytes == s.Length); internal static bool CouldBeInteger(string s) { @@ -245,18 +245,18 @@ internal static bool CouldBeInteger(string s) for (int i = isSigned ? 1 : 0; i < s.Length; i++) { char c = s[i]; - if (c < '0' | c > '9') return false; + if ((c < '0') | (c > '9')) return false; } return true; } internal static bool CouldBeInteger(ReadOnlySpan s) { - if (s.IsEmpty | s.Length > Format.MaxInt64TextLen) return false; + if (s.IsEmpty | (s.Length > Format.MaxInt64TextLen)) return false; bool isSigned = s[0] == '-'; for (int i = isSigned ? 1 : 0; i < s.Length; i++) { byte c = s[i]; - if (c < (byte)'0' | c > (byte)'9') return false; + if ((c < (byte)'0') | (c > (byte)'9')) return false; } return true; } @@ -281,7 +281,7 @@ internal static bool TryParseDouble(ReadOnlySpan s, out double value) case 4 when s[0] == '-' && TryParseInfNaN(s.Slice(1), false, out value): return true; } - return Utf8Parser.TryParse(s, out value, out int bytes) & bytes == s.Length; + return Utf8Parser.TryParse(s, out value, out int bytes) & (bytes == s.Length); static bool TryParseInfNaN(ReadOnlySpan s, bool positive, out double value) { diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs index 28f45851f..66bebc55b 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.VectorSets.cs @@ -4,9 +4,9 @@ // ReSharper disable once CheckNamespace namespace StackExchange.Redis; -/// -/// Describes functionality that is common to both standalone redis servers and redis clusters. -/// +/// +/// Vector set operations. +/// public partial interface IDatabase { // Vector Set operations diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs index 64e8e3878..ffb04f8eb 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.VectorSets.cs @@ -5,9 +5,9 @@ // ReSharper disable once CheckNamespace namespace StackExchange.Redis; -/// -/// Describes functionality that is common to both standalone redis servers and redis clusters. -/// +/// +/// Vector set operations. +/// public partial interface IDatabaseAsync { // Vector Set operations diff --git a/src/StackExchange.Redis/KeyNotification.cs b/src/StackExchange.Redis/KeyNotification.cs index 5550fe6e0..76bf87766 100644 --- a/src/StackExchange.Redis/KeyNotification.cs +++ b/src/StackExchange.Redis/KeyNotification.cs @@ -254,8 +254,9 @@ public int Database if (end <= 0) return -1; span = span.Slice(0, end); - return Utf8Parser.TryParse(span, out int database, out var bytes) - && bytes == end ? database : -1; + return Utf8Parser.TryParse(span, out int database, out var bytes) && bytes == end + ? database + : -1; } } @@ -276,7 +277,7 @@ public RedisKey GetKey() case KeyNotificationKind.KeyEvent: // Channel contains the event-type, payload contains the key byte[]? blob = _value; - if (_keyOffset != 0 & blob is not null) + if ((_keyOffset != 0) & (blob is not null)) { return blob.AsSpan(_keyOffset).ToArray(); } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs index c420b7bdc..8584b2cee 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs @@ -562,8 +562,8 @@ public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, Comma public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, When when = When.Always, CommandFlags flags = CommandFlags.None) => Inner.SortedSetAddAsync(ToInner(key), values, when, flags); - public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetAddAsync(ToInner(key), values, updateWhen, flags); + public Task SortedSetAddAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetAddAsync(ToInner(key), values, when, flags); public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, CommandFlags flags) => Inner.SortedSetAddAsync(ToInner(key), member, score, flags); @@ -571,8 +571,8 @@ public Task SortedSetAddAsync(RedisKey key, RedisValue member, double scor public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, When when = When.Always, CommandFlags flags = CommandFlags.None) => Inner.SortedSetAddAsync(ToInner(key), member, score, when, flags); - public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetAddAsync(ToInner(key), member, score, updateWhen, flags); + public Task SortedSetAddAsync(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetAddAsync(ToInner(key), member, score, when, flags); public Task SortedSetCombineAsync(SetOperation operation, RedisKey[] keys, double[]? weights = null, Aggregate aggregate = Aggregate.Sum, CommandFlags flags = CommandFlags.None) => Inner.SortedSetCombineAsync(operation, ToInner(keys), weights, aggregate, flags); @@ -670,11 +670,11 @@ public Task SortedSetRemoveRangeByValueAsync(RedisKey key, RedisValue min, public IAsyncEnumerable SortedSetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => Inner.SortedSetScanAsync(ToInner(key), pattern, pageSize, cursor, pageOffset, flags); - public Task SortedSetUpdateAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetUpdateAsync(ToInner(key), values, updateWhen, flags); + public Task SortedSetUpdateAsync(RedisKey key, SortedSetEntry[] values, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetUpdateAsync(ToInner(key), values, when, flags); - public Task SortedSetUpdateAsync(RedisKey key, RedisValue member, double score, SortedSetWhen updateWhen = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => - Inner.SortedSetUpdateAsync(ToInner(key), member, score, updateWhen, flags); + public Task SortedSetUpdateAsync(RedisKey key, RedisValue member, double score, SortedSetWhen when = SortedSetWhen.Always, CommandFlags flags = CommandFlags.None) => + Inner.SortedSetUpdateAsync(ToInner(key), member, score, when, flags); public Task SortedSetPopAsync(RedisKey key, Order order = Order.Ascending, CommandFlags flags = CommandFlags.None) => Inner.SortedSetPopAsync(ToInner(key), order, flags); @@ -709,17 +709,17 @@ public Task StreamAddAsync(RedisKey key, RedisValue streamField, Red public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => Inner.StreamAddAsync(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, flags); - public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAddAsync(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAddAsync(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamAddOptions options, CommandFlags flags = CommandFlags.None) => Inner.StreamAddAsync(ToInner(key), streamField, streamValue, options, flags); diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs index 3353c0d6c..8ca3bca48 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs @@ -583,8 +583,8 @@ public RedisValue[] SortedSetRangeByRank(RedisKey key, long start = 0, long stop Inner.SortedSetRangeByRank(ToInner(key), start, stop, order, flags); public long SortedSetRangeAndStore( - RedisKey destinationKey, RedisKey sourceKey, + RedisKey destinationKey, RedisValue start, RedisValue stop, SortedSetOrder sortedSetOrder = SortedSetOrder.ByRank, @@ -673,17 +673,17 @@ public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue str public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => Inner.StreamAdd(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, flags); - public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamField, streamValue, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamPairs, messageId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamField, streamValue, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); - public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => - Inner.StreamAdd(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, mode, flags); + public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) => + Inner.StreamAdd(ToInner(key), streamPairs, idempotentId, maxLength, useApproximateMaxLength, limit, trimMode, flags); public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamAddOptions options, CommandFlags flags = CommandFlags.None) => Inner.StreamAdd(ToInner(key), streamField, streamValue, options, flags); diff --git a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs index e7bb2044e..84c69d943 100644 --- a/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs +++ b/src/StackExchange.Redis/Maintenance/MaintenanceFaultSurface.cs @@ -4,10 +4,10 @@ namespace StackExchange.Redis; -/// +/// /// Carries maintenance context on the faults a disruption can cause, so that a timeout during an announced /// migration is distinguishable from an ordinary one. -/// +/// /// /// This follows the established pattern on these types - Commandstatus and Flags on /// , FailureType on - rather diff --git a/src/StackExchange.Redis/MessageWriter.cs b/src/StackExchange.Redis/MessageWriter.cs index 82870f1a9..0949c44ae 100644 --- a/src/StackExchange.Redis/MessageWriter.cs +++ b/src/StackExchange.Redis/MessageWriter.cs @@ -213,7 +213,7 @@ internal static void WriteMultiBulkHeader(IBufferWriter writer, long count internal static void WriteMultiBulkHeader(IBufferWriter writer, long count, RespPrefix prefix) { - if ((prefix is RespPrefix.Map or RespPrefix.Attribute) & count > 0) + if ((prefix is RespPrefix.Map or RespPrefix.Attribute) & (count > 0)) { if ((count & 1) != 0) Throw(prefix, count); count >>= 1; diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index 3ad0afb28..ee3a7bfb2 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -930,7 +930,7 @@ internal WriteResult WriteMessageTakingWriteLockSync(PhysicalConnection physical var result = WriteMessageInsideLock(physical, message); - if (result == WriteResult.Success & message.IsFlushRequiredSync) + if ((result == WriteResult.Success) & message.IsFlushRequiredSync) { physical.Flush(); } @@ -1433,7 +1433,7 @@ internal ValueTask WriteMessageTakingWriteLockAsync(PhysicalConnect if (!gotLock) return new ValueTask(TimedOutBeforeWrite(message)); } var result = WriteMessageInsideLock(physical, message); - if (result == WriteResult.Success & message.IsFlushRequiredAsync) + if ((result == WriteResult.Success) & message.IsFlushRequiredAsync) { physical.Flush(); } diff --git a/src/StackExchange.Redis/PhysicalConnection.Read.cs b/src/StackExchange.Redis/PhysicalConnection.Read.cs index 8aca7e41c..4fe44624f 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Read.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Read.cs @@ -291,7 +291,7 @@ private bool CommitAndParseFrames(int bytesRead) Debug.Assert( state is { - IsComplete: true, TotalBytes: >= RespScanState.MinBytes, Prefix: not RespPrefix.None + IsComplete: true, TotalBytes: >= RespScanState.MinBytes, Prefix: not RespPrefix.None, }, "Invalid RESP read state"); @@ -340,7 +340,7 @@ state is Debug.Assert( state is { - IsComplete: true, TotalBytes: >= RespScanState.MinBytes, Prefix: not RespPrefix.None + IsComplete: true, TotalBytes: >= RespScanState.MinBytes, Prefix: not RespPrefix.None, }, "Invalid RESP read state"); @@ -442,7 +442,7 @@ private void OnResponseFrame(RespPrefix prefix, ReadOnlySpan frame, ref IM switch (prefix) { case RespPrefix.Push: // explicit RESP3 push message - case RespPrefix.Array when (_protocol is RedisProtocol.Resp2 & connectionType is ConnectionType.Subscription) + case RespPrefix.Array when ((_protocol is RedisProtocol.Resp2) & (connectionType is ConnectionType.Subscription)) && !IsArrayPong(frame): // could be a RESP2 pub/sub payload // out-of-band; pub/sub etc switch (OnOutOfBand(frame, ref memoryOwner)) diff --git a/src/StackExchange.Redis/Profiling/ProfiledCommand.cs b/src/StackExchange.Redis/Profiling/ProfiledCommand.cs index 84ea23f08..ecaaee7b5 100644 --- a/src/StackExchange.Redis/Profiling/ProfiledCommand.cs +++ b/src/StackExchange.Redis/Profiling/ProfiledCommand.cs @@ -120,11 +120,16 @@ public void SetCompleted() } } - public override string ToString() => -$@"{Command} (DB: {Db}, Flags: {Flags}) + public override string ToString() + { + var retransmission = RetransmissionOf != null + ? @" + RetransmissionOf = " + RetransmissionOf + : ""; + return $@"{Command} (DB: {Db}, Flags: {Flags}) EndPoint = {EndPoint} ({ConnectionType}) Created = {CommandCreated:HH:mm:ss.ffff} - ElapsedTime = {ElapsedTime.TotalMilliseconds} ms (CreationToEnqueued: {CreationToEnqueued.TotalMilliseconds} ms, EnqueuedToSending: {EnqueuedToSending.TotalMilliseconds} ms, SentToResponse: {SentToResponse.TotalMilliseconds} ms, ResponseToCompletion = {ResponseToCompletion.TotalMilliseconds} ms){(RetransmissionOf != null ? @" - RetransmissionOf = " + RetransmissionOf : "")}"; + ElapsedTime = {ElapsedTime.TotalMilliseconds} ms (CreationToEnqueued: {CreationToEnqueued.TotalMilliseconds} ms, EnqueuedToSending: {EnqueuedToSending.TotalMilliseconds} ms, SentToResponse: {SentToResponse.TotalMilliseconds} ms, ResponseToCompletion = {ResponseToCompletion.TotalMilliseconds} ms){retransmission}"; + } } } diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index 8cc5c21b9..6d203b53f 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -33,3 +33,7 @@ static StackExchange.Redis.Availability.HealthCheckProbe.LagAware(StackExchange. StackExchange.Redis.Availability.LagAwareOptions.ClientCertificate.get -> System.Security.Cryptography.X509Certificates.X509Certificate2? StackExchange.Redis.Availability.LagAwareOptions.ClientCertificate.set -> void StackExchange.Redis.Availability.LagAwareOptions.SetUserPfxCertificate(string! userCertificatePath, string? password = null) -> void +*REMOVED*override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! muxer, System.Action! log) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.AzureManagedRedisOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! multiplexer, System.Action! log) -> System.Threading.Tasks.Task! +*REMOVED*override StackExchange.Redis.Configuration.AzureOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! muxer, System.Action! log) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.AzureOptionsProvider.AfterConnectAsync(StackExchange.Redis.ConnectionMultiplexer! multiplexer, System.Action! log) -> System.Threading.Tasks.Task! diff --git a/src/StackExchange.Redis/RedisChannel.cs b/src/StackExchange.Redis/RedisChannel.cs index b917a2a62..a9c3c10bc 100644 --- a/src/StackExchange.Redis/RedisChannel.cs +++ b/src/StackExchange.Redis/RedisChannel.cs @@ -476,7 +476,7 @@ private static Span AppendAndAdvance(Span target, scoped ReadOnlySpa private static RedisChannel BuildKeySpaceChannel(in RedisKey key, int? database, RedisChannelOptions options, ReadOnlySpan suffix, bool appendStar, bool allowKeyPatterns, bool subkey) { int fullKeyLength = key.TotalLength() + suffix.Length + (appendStar ? 1 : 0); - if (appendStar & (options & RedisChannelOptions.Pattern) == 0) throw new ArgumentNullException(nameof(key)); + if (appendStar & ((options & RedisChannelOptions.Pattern) == 0)) throw new ArgumentNullException(nameof(key)); if (fullKeyLength == 0) throw new ArgumentOutOfRangeException(nameof(key)); var db = AppendDatabase(stackalloc byte[DatabaseScratchBufferSize], database, options); diff --git a/src/StackExchange.Redis/RedisDatabase.cs b/src/StackExchange.Redis/RedisDatabase.cs index cfe24f7aa..3c9c28918 100644 --- a/src/StackExchange.Redis/RedisDatabase.cs +++ b/src/StackExchange.Redis/RedisDatabase.cs @@ -36,7 +36,8 @@ DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) // to fold in their own flag. Cluster is intrinsic to the underlying connection, so it lives here. private protected virtual DatabaseFeatureFlags GetDatabaseFeatures() => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster - ? DatabaseFeatureFlags.Cluster : DatabaseFeatureFlags.None; + ? DatabaseFeatureFlags.Cluster + : DatabaseFeatureFlags.None; public IBatch CreateBatch(object? asyncState) { @@ -126,9 +127,9 @@ public Task GeoRemoveAsync(RedisKey key, RedisValue member, CommandFlags f return ExecuteSync(msg, ResultProcessor.NullableDouble); } - public Task GeoDistanceAsync(RedisKey key, RedisValue value0, RedisValue value1, GeoUnit unit = GeoUnit.Meters, CommandFlags flags = CommandFlags.None) + public Task GeoDistanceAsync(RedisKey key, RedisValue member1, RedisValue member2, GeoUnit unit = GeoUnit.Meters, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.GEODIST, key, value0, value1, StackExchange.Redis.GeoPosition.GetRedisUnit(unit)); + var msg = Message.Create(Database, flags, RedisCommand.GEODIST, key, member1, member2, StackExchange.Redis.GeoPosition.GetRedisUnit(unit)); return ExecuteAsync(msg, ResultProcessor.NullableDouble); } @@ -860,28 +861,32 @@ public Task HashGetAsync(RedisKey key, RedisValue[] hashFields, Co public long HashIncrement(RedisKey key, RedisValue hashField, long value = 1, CommandFlags flags = CommandFlags.None) { var msg = value == 0 && (flags & CommandFlags.FireAndForget) != 0 - ? null : Message.Create(Database, flags, RedisCommand.HINCRBY, key, hashField, value); + ? null + : Message.Create(Database, flags, RedisCommand.HINCRBY, key, hashField, value); return ExecuteSync(msg, ResultProcessor.Int64); } public double HashIncrement(RedisKey key, RedisValue hashField, double value, CommandFlags flags = CommandFlags.None) { var msg = value == 0 && (flags & CommandFlags.FireAndForget) != 0 - ? null : Message.Create(Database, flags, RedisCommand.HINCRBYFLOAT, key, hashField, value); + ? null + : Message.Create(Database, flags, RedisCommand.HINCRBYFLOAT, key, hashField, value); return ExecuteSync(msg, ResultProcessor.Double); } public Task HashIncrementAsync(RedisKey key, RedisValue hashField, long value = 1, CommandFlags flags = CommandFlags.None) { var msg = value == 0 && (flags & CommandFlags.FireAndForget) != 0 - ? null : Message.Create(Database, flags, RedisCommand.HINCRBY, key, hashField, value); + ? null + : Message.Create(Database, flags, RedisCommand.HINCRBY, key, hashField, value); return ExecuteAsync(msg, ResultProcessor.Int64); } public Task HashIncrementAsync(RedisKey key, RedisValue hashField, double value, CommandFlags flags = CommandFlags.None) { var msg = value == 0 && (flags & CommandFlags.FireAndForget) != 0 - ? null : Message.Create(Database, flags, RedisCommand.HINCRBYFLOAT, key, hashField, value); + ? null + : Message.Create(Database, flags, RedisCommand.HINCRBYFLOAT, key, hashField, value); return ExecuteAsync(msg, ResultProcessor.Double); } @@ -1323,9 +1328,9 @@ public Task KeyExpireAsync(RedisKey key, TimeSpan? expiry, ExpireWhen when return ExecuteAsync(msg, ResultProcessor.Boolean, server: server); } - public Task KeyExpireAsync(RedisKey key, DateTime? expire, ExpireWhen when, CommandFlags flags = CommandFlags.None) + public Task KeyExpireAsync(RedisKey key, DateTime? expiry, ExpireWhen when, CommandFlags flags = CommandFlags.None) { - var msg = GetExpiryMessage(key, flags, expire, when, out ServerEndPoint? server); + var msg = GetExpiryMessage(key, flags, expiry, when, out ServerEndPoint? server); return ExecuteAsync(msg, ResultProcessor.Boolean, server: server); } @@ -1953,39 +1958,39 @@ public Task LockTakeAsync(RedisKey key, RedisValue value, TimeSpan expiry, return StringSetAsync(key, value, expiry, When.NotExists, flags); } - public string? StringLongestCommonSubsequence(RedisKey key1, RedisKey key2, CommandFlags flags = CommandFlags.None) + public string? StringLongestCommonSubsequence(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.LCS, key1, key2); + var msg = Message.Create(Database, flags, RedisCommand.LCS, first, second); return ExecuteSync(msg, ResultProcessor.String); } - public Task StringLongestCommonSubsequenceAsync(RedisKey key1, RedisKey key2, CommandFlags flags = CommandFlags.None) + public Task StringLongestCommonSubsequenceAsync(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.LCS, key1, key2); + var msg = Message.Create(Database, flags, RedisCommand.LCS, first, second); return ExecuteAsync(msg, ResultProcessor.String); } - public long StringLongestCommonSubsequenceLength(RedisKey key1, RedisKey key2, CommandFlags flags = CommandFlags.None) + public long StringLongestCommonSubsequenceLength(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.LCS, key1, key2, RedisLiterals.LEN); + var msg = Message.Create(Database, flags, RedisCommand.LCS, first, second, RedisLiterals.LEN); return ExecuteSync(msg, ResultProcessor.Int64); } - public Task StringLongestCommonSubsequenceLengthAsync(RedisKey key1, RedisKey key2, CommandFlags flags = CommandFlags.None) + public Task StringLongestCommonSubsequenceLengthAsync(RedisKey first, RedisKey second, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.LCS, key1, key2, RedisLiterals.LEN); + var msg = Message.Create(Database, flags, RedisCommand.LCS, first, second, RedisLiterals.LEN); return ExecuteAsync(msg, ResultProcessor.Int64); } - public LCSMatchResult StringLongestCommonSubsequenceWithMatches(RedisKey key1, RedisKey key2, long minSubMatchLength = 0, CommandFlags flags = CommandFlags.None) + public LCSMatchResult StringLongestCommonSubsequenceWithMatches(RedisKey first, RedisKey second, long minLength = 0, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.LCS, key1, key2, RedisLiterals.IDX, RedisLiterals.MINMATCHLEN, minSubMatchLength, RedisLiterals.WITHMATCHLEN); + var msg = Message.Create(Database, flags, RedisCommand.LCS, first, second, RedisLiterals.IDX, RedisLiterals.MINMATCHLEN, minLength, RedisLiterals.WITHMATCHLEN); return ExecuteSync(msg, ResultProcessor.LCSMatchResult); } - public Task StringLongestCommonSubsequenceWithMatchesAsync(RedisKey key1, RedisKey key2, long minSubMatchLength = 0, CommandFlags flags = CommandFlags.None) + public Task StringLongestCommonSubsequenceWithMatchesAsync(RedisKey first, RedisKey second, long minLength = 0, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.LCS, key1, key2, RedisLiterals.IDX, RedisLiterals.MINMATCHLEN, minSubMatchLength, RedisLiterals.WITHMATCHLEN); + var msg = Message.Create(Database, flags, RedisCommand.LCS, first, second, RedisLiterals.IDX, RedisLiterals.MINMATCHLEN, minLength, RedisLiterals.WITHMATCHLEN); return ExecuteAsync(msg, ResultProcessor.LCSMatchResult); } @@ -3015,16 +3020,16 @@ public Task StreamAcknowledgeAndDeleteAsync(RedisKey key, Re public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => StreamAdd(key, streamField, streamValue, messageId, maxLength, useApproximateMaxLength, null, StreamTrimMode.KeepReferences, flags); - public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, new NameValueEntry(streamField, streamValue), flags); return ExecuteSync(msg, ResultProcessor.RedisValue); } - public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, new NameValueEntry(streamField, streamValue), flags); return ExecuteSync(msg, ResultProcessor.RedisValue); } @@ -3039,16 +3044,16 @@ public RedisValue StreamAdd(RedisKey key, RedisValue streamField, RedisValue str public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => StreamAddAsync(key, streamField, streamValue, messageId, maxLength, useApproximateMaxLength, null, StreamTrimMode.KeepReferences, flags); - public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, new NameValueEntry(streamField, streamValue), flags); return ExecuteAsync(msg, ResultProcessor.RedisValue); } - public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public Task StreamAddAsync(RedisKey key, RedisValue streamField, RedisValue streamValue, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, new NameValueEntry(streamField, streamValue), flags); return ExecuteAsync(msg, ResultProcessor.RedisValue); } @@ -3063,16 +3068,16 @@ public Task StreamAddAsync(RedisKey key, RedisValue streamField, Red public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => StreamAdd(key, streamPairs, messageId, maxLength, useApproximateMaxLength, null, StreamTrimMode.KeepReferences, flags); - public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, streamPairs, flags); return ExecuteSync(msg, ResultProcessor.RedisValue); } - public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, streamPairs, flags); return ExecuteSync(msg, ResultProcessor.RedisValue); } @@ -3087,16 +3092,16 @@ public RedisValue StreamAdd(RedisKey key, NameValueEntry[] streamPairs, StreamAd public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId, int? maxLength, bool useApproximateMaxLength, CommandFlags flags) => StreamAddAsync(key, streamPairs, messageId, maxLength, useApproximateMaxLength, null, StreamTrimMode.KeepReferences, flags); - public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, RedisValue? messageId = null, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(messageId, StreamIdempotentId.Empty, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, streamPairs, flags); return ExecuteAsync(msg, ResultProcessor.RedisValue); } - public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode mode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) + public Task StreamAddAsync(RedisKey key, NameValueEntry[] streamPairs, StreamIdempotentId idempotentId, long? maxLength = null, bool useApproximateMaxLength = false, long? limit = null, StreamTrimMode trimMode = StreamTrimMode.KeepReferences, CommandFlags flags = CommandFlags.None) { - var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, mode); + var options = LegacyStreamAddOptions(null, in idempotentId, maxLength, useApproximateMaxLength, limit, trimMode); var msg = GetStreamAddMessage(key, in options, streamPairs, flags); return ExecuteAsync(msg, ResultProcessor.RedisValue); } @@ -4099,7 +4104,8 @@ public long StringIncrement(RedisKey key, long value = 1, CommandFlags flags = C public double StringIncrement(RedisKey key, double value, CommandFlags flags = CommandFlags.None) { var msg = value == 0 && (flags & CommandFlags.FireAndForget) != 0 - ? null : Message.Create(Database, flags, RedisCommand.INCRBYFLOAT, key, value); + ? null + : Message.Create(Database, flags, RedisCommand.INCRBYFLOAT, key, value); return ExecuteSync(msg, ResultProcessor.Double); } @@ -4112,7 +4118,8 @@ public Task StringIncrementAsync(RedisKey key, long value = 1, CommandFlag public Task StringIncrementAsync(RedisKey key, double value, CommandFlags flags = CommandFlags.None) { var msg = value == 0 && (flags & CommandFlags.FireAndForget) != 0 - ? null : Message.Create(Database, flags, RedisCommand.INCRBYFLOAT, key, value); + ? null + : Message.Create(Database, flags, RedisCommand.INCRBYFLOAT, key, value); return ExecuteAsync(msg, ResultProcessor.Double); } @@ -4200,9 +4207,9 @@ public bool StringSetBit(RedisKey key, long offset, bool bit, CommandFlags flags return ExecuteSync(msg, ResultProcessor.Boolean); } - public Task StringSetBitAsync(RedisKey key, long offset, bool value, CommandFlags flags = CommandFlags.None) + public Task StringSetBitAsync(RedisKey key, long offset, bool bit, CommandFlags flags = CommandFlags.None) { - var msg = Message.Create(Database, flags, RedisCommand.SETBIT, key, offset, value); + var msg = Message.Create(Database, flags, RedisCommand.SETBIT, key, offset, bit); return ExecuteAsync(msg, ResultProcessor.Boolean); } @@ -4943,12 +4950,14 @@ private Message GetSortedSetRangeByScoreMessage(RedisKey key, double start, doub RedisValue from = GetRange(start, exclude, true), to = GetRange(stop, exclude, false); if (withScores) { - return unlimited ? Message.Create(Database, flags, command, key, from, to, RedisLiterals.WITHSCORES) + return unlimited + ? Message.Create(Database, flags, command, key, from, to, RedisLiterals.WITHSCORES) : Message.Create(Database, flags, command, key, new[] { from, to, RedisLiterals.WITHSCORES, RedisLiterals.LIMIT, skip, take }); } else { - return unlimited ? Message.Create(Database, flags, command, key, from, to) + return unlimited + ? Message.Create(Database, flags, command, key, from, to) : Message.Create(Database, flags, command, key, new[] { from, to, RedisLiterals.LIMIT, skip, take }); } } @@ -5728,7 +5737,7 @@ internal Message GetStringSetMessage( expiry.GetTokenCount(allowEnx: false); - if (when is When.Always & expiry.IsRelative) + if ((when is When.Always) & expiry.IsRelative) { // special case to SETEX/PSETEX return expiry.IsSeconds @@ -6059,7 +6068,7 @@ private static int RemoveDbIfNotRequired(int suggestedDb, string adhocCommand, o { knownCommand = RedisCommand.UNKNOWN; } - if ((knownCommand is not RedisCommand.UNKNOWN & suggestedDb >= 0) && !Message.RequiresDatabase(knownCommand)) + if (((knownCommand is not RedisCommand.UNKNOWN) & (suggestedDb >= 0)) && !Message.RequiresDatabase(knownCommand)) { // strip the DB; historically we didn't enforce this when IDatabase was // used to issue known commands as strings, so: don't complain now @@ -6153,7 +6162,7 @@ private static int RemoveDbIfNotRequired(int suggestedDb, string adhocCommand, o { knownCommand = RedisCommand.UNKNOWN; } - if ((knownCommand is not RedisCommand.UNKNOWN & suggestedDb >= 0) && !Message.RequiresDatabase(knownCommand)) + if (((knownCommand is not RedisCommand.UNKNOWN) & (suggestedDb >= 0)) && !Message.RequiresDatabase(knownCommand)) { // strip the DB; historically we didn't enforce this when IDatabase was // used to issue known commands as strings, so: don't complain now diff --git a/src/StackExchange.Redis/RedisErrorKind.cs b/src/StackExchange.Redis/RedisErrorKind.cs index e4403778b..ccee997f0 100644 --- a/src/StackExchange.Redis/RedisErrorKind.cs +++ b/src/StackExchange.Redis/RedisErrorKind.cs @@ -201,7 +201,7 @@ internal static bool TryParse(ReadOnlySpan value, out RedisErrorKind kind) if (TryParseFirstTokenCI(firstToken, out kind)) { // check for more specific "ERR ..." scenarios - if (kind is RedisErrorKind.UnknownError & space > 0) + if ((kind is RedisErrorKind.UnknownError) & (space > 0)) { // get the message text after "ERR " value = value.Slice(space + 1); diff --git a/src/StackExchange.Redis/RedisKey.cs b/src/StackExchange.Redis/RedisKey.cs index 85f9ced59..31abd7c9a 100644 --- a/src/StackExchange.Redis/RedisKey.cs +++ b/src/StackExchange.Redis/RedisKey.cs @@ -342,9 +342,11 @@ internal static RedisKey WithPrefix(byte[]? prefix, RedisKey value) } int aLen = a?.Length ?? 0, - bLen = b == null ? 0 : (b is string bString - ? Encoding.UTF8.GetByteCount(bString) - : ((byte[])b).Length), + bLen = b == null + ? 0 + : b is string bString + ? Encoding.UTF8.GetByteCount(bString) + : ((byte[])b).Length, cLen = c?.Length ?? 0; var result = new byte[aLen + bLen + cLen]; diff --git a/src/StackExchange.Redis/RedisResult.cs b/src/StackExchange.Redis/RedisResult.cs index 1d496468f..883e4738f 100644 --- a/src/StackExchange.Redis/RedisResult.cs +++ b/src/StackExchange.Redis/RedisResult.cs @@ -44,8 +44,9 @@ public static RedisResult Create(RedisValue[] values) /// The explicit data type. /// new . public static RedisResult Create(RedisValue[] values, ResultType resultType) => - values == null ? NullArray : values.Length == 0 ? EmptyArray(resultType) : - new ArrayRedisResult(Array.ConvertAll(values, value => new SingleRedisResult(value, null)), resultType); + values == null ? NullArray : + values.Length == 0 ? EmptyArray(resultType) : + new ArrayRedisResult(Array.ConvertAll(values, value => new SingleRedisResult(value, null)), resultType); /// /// Create a new RedisResult representing an array of values. @@ -428,10 +429,11 @@ internal override bool AsBoolean() } internal override byte[][]? AsByteArrayArray() - => IsNull ? null - : _value!.Length == 0 - ? Array.Empty() - : Array.ConvertAll(_value, x => x.AsByteArray()!); + => IsNull + ? null + : _value!.Length == 0 + ? Array.Empty() + : Array.ConvertAll(_value, x => x.AsByteArray()!); private bool IsSingleton => _value?.Length == 1; private bool IsEmpty => _value?.Length == 0; @@ -442,9 +444,11 @@ internal override double AsDouble() } internal override double[]? AsDoubleArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsDouble()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsDouble()); internal override int AsInt32() { @@ -453,9 +457,11 @@ internal override int AsInt32() } internal override int[]? AsInt32Array() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsInt32()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsInt32()); internal override long AsInt64() { @@ -469,14 +475,18 @@ internal override ulong AsUInt64() } internal override long[]? AsInt64Array() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsInt64()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsInt64()); internal override ulong[]? AsUInt64Array() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsUInt64()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsUInt64()); internal override bool? AsNullableBoolean() { @@ -514,9 +524,11 @@ internal override RedisKey AsRedisKey() } internal override RedisKey[]? AsRedisKeyArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsRedisKey()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsRedisKey()); internal override RedisResult[]? AsRedisResultArray() => _value; @@ -527,9 +539,11 @@ internal override RedisValue AsRedisValue() } internal override RedisValue[]? AsRedisValueArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsRedisValue()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsRedisValue()); internal override string? AsString() { @@ -538,9 +552,11 @@ internal override RedisValue AsRedisValue() } internal override string?[]? AsStringArray() - => IsNull ? null - : IsEmpty ? Array.Empty() - : Array.ConvertAll(_value!, x => x.AsString()); + => IsNull + ? null + : IsEmpty + ? Array.Empty() + : Array.ConvertAll(_value!, x => x.AsString()); } /// diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index 94fb73065..e6d9cb000 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -587,14 +587,16 @@ public Task StringGetAsync(int db, RedisKey key, CommandFlags flags public RedisChannel[] SubscriptionChannels(RedisChannel pattern = default, CommandFlags flags = CommandFlags.None) { - var msg = pattern.IsNullOrEmpty ? Message.Create(-1, flags, RedisCommand.PUBSUB, RedisLiterals.CHANNELS) + var msg = pattern.IsNullOrEmpty + ? Message.Create(-1, flags, RedisCommand.PUBSUB, RedisLiterals.CHANNELS) : Message.Create(-1, flags, RedisCommand.PUBSUB, RedisLiterals.CHANNELS, pattern); return ExecuteSync(msg, ResultProcessor.RedisChannelArrayLiteral, defaultValue: Array.Empty()); } public Task SubscriptionChannelsAsync(RedisChannel pattern = default, CommandFlags flags = CommandFlags.None) { - var msg = pattern.IsNullOrEmpty ? Message.Create(-1, flags, RedisCommand.PUBSUB, RedisLiterals.CHANNELS) + var msg = pattern.IsNullOrEmpty + ? Message.Create(-1, flags, RedisCommand.PUBSUB, RedisLiterals.CHANNELS) : Message.Create(-1, flags, RedisCommand.PUBSUB, RedisLiterals.CHANNELS, pattern); return ExecuteAsync(msg, ResultProcessor.RedisChannelArrayLiteral, defaultValue: Array.Empty()); } diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index ef9d42d22..d0741e5f5 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -332,7 +332,7 @@ private bool UnregisterSubscription(in RedisChannel channel, Action value) /// Creates a new from an . /// /// The to convert to a . - public static implicit operator RedisValue(bool? value) => value == null ? Null + public static implicit operator RedisValue(bool? value) => value == null + ? Null : new(value.GetValueOrDefault() ? 1 : 0); /// diff --git a/src/StackExchange.Redis/RenderedArgs.cs b/src/StackExchange.Redis/RenderedArgs.cs index 43389744d..3221d2faa 100644 --- a/src/StackExchange.Redis/RenderedArgs.cs +++ b/src/StackExchange.Redis/RenderedArgs.cs @@ -124,7 +124,8 @@ public static RenderedArgs Create(ReadOnlySpan keys, ReadOnlySpan? pool) => new RenderedArgs { - _buffer = length == 0 ? Array.Empty() + _buffer = length == 0 + ? Array.Empty() : pool is null ? ArrayPool.Shared.Rent(length) : pool.Rent(length), _count = count, _length = length, diff --git a/src/StackExchange.Redis/RespReaderExtensions.cs b/src/StackExchange.Redis/RespReaderExtensions.cs index 1048d5e2c..587ef32ec 100644 --- a/src/StackExchange.Redis/RespReaderExtensions.cs +++ b/src/StackExchange.Redis/RespReaderExtensions.cs @@ -44,8 +44,7 @@ public static RedisValue ReadRedisValue(this in RespReader reader) // discontiguous payload. if (length <= RedisValue.MaxInlineBytes) { - return RedisValue.FromRaw(reader.TryGetSpan(out var buffer) ? - buffer : reader.Buffer(stackalloc byte[RedisValue.MaxInlineBytes])); + return RedisValue.FromRaw(reader.TryGetSpan(out var buffer) ? buffer : reader.Buffer(stackalloc byte[RedisValue.MaxInlineBytes])); } // Longer payloads: prefer a compact numeric storage kind when the text is the *canonical* @@ -148,7 +147,7 @@ static bool Failure(out RedisValue value) } // integer: canonical exactly when the round-trip text length matches (rules out leading zeros, a // leading '+', "-0", trailing junk, etc.) - so no need to re-emit and compare bytes - if (span.IsEmpty | span.Length > MaxCanonicalLength) return Failure(out value); + if (span.IsEmpty | (span.Length > MaxCanonicalLength)) return Failure(out value); // restrict to *just* basic number tokens, tracking the two facts that let us pick a single parse: // whether a '-' appeared (so we know whether the integer is signed), and whether a '.'/'e'/'E' diff --git a/src/StackExchange.Redis/ResultBox.cs b/src/StackExchange.Redis/ResultBox.cs index eeacae0f0..26a72111b 100644 --- a/src/StackExchange.Redis/ResultBox.cs +++ b/src/StackExchange.Redis/ResultBox.cs @@ -42,7 +42,7 @@ internal abstract class SimpleResultBox : IResultBox bool IResultBox.IsFaulted => _exception != null; bool IResultBox.IsCompleted => _completed; Exception? IResultBox.Fault => _exception; - void IResultBox.SetException(Exception exception) => _exception = exception ?? CancelledException; + void IResultBox.SetException(Exception ex) => _exception = ex ?? CancelledException; void IResultBox.Cancel() => _exception = CancelledException; void IResultBox.ActivateContinuations() @@ -131,7 +131,7 @@ private TaskResultBox(object? asyncState, TaskCreationOptions creationOptions) : void IResultBox.SetResult(T value) => _value = value; - T? IResultBox.GetResult(out Exception? ex, bool unused) + T? IResultBox.GetResult(out Exception? ex, bool canRecycle) { ex = _exception; return _value; diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index e985b10f2..9b2180301 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -1277,7 +1277,8 @@ when helloFromInteractive // a known non-Redis server: "version" is the vendor's own version, and the Redis API version // (if the server reports one at all) comes from INFO's redis_version var productVersion = vendorVariant == fromServer && !string.IsNullOrWhiteSpace(vendorVersion) - ? vendorVersion! : helloVersion.IsScalar ? helloVersion.ReadString() : null; + ? vendorVersion! + : helloVersion.IsScalar ? helloVersion.ReadString() : null; if (!string.IsNullOrWhiteSpace(productVersion)) { server.SetProductVariant(fromServer, productVersion!); diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index eb9d9fa02..0768dbcb7 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -111,8 +111,8 @@ public ServerEndPoint(ConnectionMultiplexer multiplexer, EndPoint endpoint, Serv private RedisServer? _defaultServer; public RedisServer GetRedisServer(object? asyncState) => asyncState is null - ? (_defaultServer ??= new RedisServer(this, null)) // reuse and memoize - : new RedisServer(this, asyncState); + ? (_defaultServer ??= new RedisServer(this, null)) // reuse and memoize + : new RedisServer(this, asyncState); public EndPoint EndPoint { get; } @@ -249,7 +249,10 @@ public bool ReplicaReadOnly public bool AllowReplicaWrites { - get => allowReplicaWrites; + get + { + return allowReplicaWrites; + } set { allowReplicaWrites = value; @@ -789,7 +792,10 @@ internal void FlushScriptCache() private string? runId; internal string? RunId { - get => runId; + get + { + return runId; + } set { // We only care about changes diff --git a/src/StackExchange.Redis/ServerSelectionStrategy.cs b/src/StackExchange.Redis/ServerSelectionStrategy.cs index a3ea96c2d..7fed58e53 100644 --- a/src/StackExchange.Redis/ServerSelectionStrategy.cs +++ b/src/StackExchange.Redis/ServerSelectionStrategy.cs @@ -113,7 +113,8 @@ public int HashSlot(in RedisChannel channel) ReadOnlySpan routingSpan = channel.RoutingSpan; byte[] prefix; return channel.IgnoreChannelPrefix || (prefix = ChannelPrefix).Length == 0 - ? GetClusterSlot(routingSpan) : GetClusterSlotWithPrefix(prefix, routingSpan); + ? GetClusterSlot(routingSpan) + : GetClusterSlotWithPrefix(prefix, routingSpan); static int GetClusterSlotWithPrefix(byte[] prefixRaw, ReadOnlySpan routingSpan) { diff --git a/src/StackExchange.Redis/Subscription.cs b/src/StackExchange.Redis/Subscription.cs index cb924a29a..767ab0a14 100644 --- a/src/StackExchange.Redis/Subscription.cs +++ b/src/StackExchange.Redis/Subscription.cs @@ -153,7 +153,7 @@ public bool Remove(Action? handler, ChannelMessageQueu ChannelMessageQueue.Remove(ref _queues, queue); } - return _handlers == null & _queues == null; + return (_handlers == null) & (_queues == null); } public ICompletable? ForInvoke(in RedisChannel channel, in RedisValue message, out ChannelMessageQueue? queues) @@ -445,7 +445,7 @@ internal override int EnsureSubscribedToServer( { // exclude sentinel, and only use replicas if we're explicitly asking for them bool useReplica = (Flags & CommandFlags.DemandReplica) != 0; - bool shouldBeConnected = server.ServerType != ServerType.Sentinel & server.IsReplica == useReplica; + bool shouldBeConnected = (server.ServerType != ServerType.Sentinel) & (server.IsReplica == useReplica); if (shouldBeConnected == IsConnectedTo(server.EndPoint)) { return null; diff --git a/src/StackExchange.Redis/ValueCondition.cs b/src/StackExchange.Redis/ValueCondition.cs index f035551b6..d1579b71d 100644 --- a/src/StackExchange.Redis/ValueCondition.cs +++ b/src/StackExchange.Redis/ValueCondition.cs @@ -231,9 +231,9 @@ public static ValueCondition ParseDigest(ReadOnlySpan digest) private static byte ParseNibble(int b) { - if (b >= '0' & b <= '9') return (byte)(b - '0'); - if (b >= 'a' & b <= 'f') return (byte)(b - 'a' + 10); - if (b >= 'A' & b <= 'F') return (byte)(b - 'A' + 10); + if ((b >= '0') & (b <= '9')) return (byte)(b - '0'); + if ((b >= 'a') & (b <= 'f')) return (byte)(b - 'a' + 10); + if ((b >= 'A') & (b <= 'F')) return (byte)(b - 'A' + 10); return ThrowInvalidBytes(); static byte ThrowInvalidBytes() => throw new ArgumentException("Invalid digest bytes"); @@ -264,9 +264,9 @@ public static ValueCondition ParseDigest(ReadOnlySpan digest) static byte ToNibble(int b) { - if (b >= '0' & b <= '9') return (byte)(b - '0'); - if (b >= 'a' & b <= 'f') return (byte)(b - 'a' + 10); - if (b >= 'A' & b <= 'F') return (byte)(b - 'A' + 10); + if ((b >= '0') & (b <= '9')) return (byte)(b - '0'); + if ((b >= 'a') & (b <= 'f')) return (byte)(b - 'a' + 10); + if ((b >= 'A') & (b <= 'F')) return (byte)(b - 'A' + 10); return ThrowInvalidBytes(); } diff --git a/src/StackExchange.Redis/VectorSetSimilaritySearchMessage.cs b/src/StackExchange.Redis/VectorSetSimilaritySearchMessage.cs index 1c495cde5..816ce6600 100644 --- a/src/StackExchange.Redis/VectorSetSimilaritySearchMessage.cs +++ b/src/StackExchange.Redis/VectorSetSimilaritySearchMessage.cs @@ -30,8 +30,7 @@ internal sealed class VectorSetSimilaritySearchBySingleVectorMessage( int searchExplorationFactor, string? filterExpression, int maxFilteringEffort, - bool useFp32) : VectorSetSimilaritySearchMessage(db, flags, vsimFlags, key, count, epsilon, - searchExplorationFactor, filterExpression, maxFilteringEffort, useFp32) + bool useFp32) : VectorSetSimilaritySearchMessage(db, flags, vsimFlags, key, count, epsilon, searchExplorationFactor, filterExpression, maxFilteringEffort, useFp32) { internal override int GetSearchTargetArgCount() => UseFp32 ? 2 : (2 + vector.Length); // FP32 {vector} or VALUES {num} {vector} @@ -67,8 +66,7 @@ internal sealed class VectorSetSimilaritySearchByMemberMessage( int searchExplorationFactor, string? filterExpression, int maxFilteringEffort, - bool useFp32) : VectorSetSimilaritySearchMessage(db, flags, vsimFlags, key, count, epsilon, - searchExplorationFactor, filterExpression, maxFilteringEffort, useFp32) + bool useFp32) : VectorSetSimilaritySearchMessage(db, flags, vsimFlags, key, count, epsilon, searchExplorationFactor, filterExpression, maxFilteringEffort, useFp32) { internal override int GetSearchTargetArgCount() => 2; // ELE {member} diff --git a/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs b/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs index c48004d8f..adfeefc08 100644 --- a/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs +++ b/src/StackExchange.Redis/VectorSetSimilaritySearchRequest.cs @@ -92,7 +92,10 @@ private void SetFlag(VsimFlags flag, bool value) /// public int? Count { - get => HasFlag(VsimFlags.Count) ? _count : null; + get + { + return HasFlag(VsimFlags.Count) ? _count : null; + } set { if (value.HasValue) @@ -130,7 +133,10 @@ public bool WithAttributes /// public double? Epsilon { - get => HasFlag(VsimFlags.Epsilon) ? _epsilon : null; + get + { + return HasFlag(VsimFlags.Epsilon) ? _epsilon : null; + } set { if (value.HasValue) @@ -150,7 +156,10 @@ public double? Epsilon /// public int? SearchExplorationFactor { - get => HasFlag(VsimFlags.SearchExplorationFactor) ? _searchExplorationFactor : null; + get + { + return HasFlag(VsimFlags.SearchExplorationFactor) ? _searchExplorationFactor : null; + } set { if (value.HasValue) @@ -170,7 +179,10 @@ public int? SearchExplorationFactor /// public int? MaxFilteringEffort { - get => HasFlag(VsimFlags.MaxFilteringEffort) ? _maxFilteringEffort : null; + get + { + return HasFlag(VsimFlags.MaxFilteringEffort) ? _maxFilteringEffort : null; + } set { if (value.HasValue) @@ -192,7 +204,10 @@ public int? MaxFilteringEffort /// public string? FilterExpression { - get => _filterExpression; + get + { + return _filterExpression; + } set { _filterExpression = value; diff --git a/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs b/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs index b70d215ee..772ed6823 100644 --- a/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs +++ b/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs @@ -131,4 +131,27 @@ public async Task ConditionTest() i = (int)raw.StringGet(prefix + "i"); Assert.Equal(1, i); } + + [Fact] + public async Task SortedSetRangeAndStoreUsesTheRightKeys() + { + // the sync prefixed overload declared its first two parameters as (destination, source) against the + // interface's (source, destination) and then forwarded them by name, so it read from the destination + // and wrote into (and so wiped) the source + await using var conn = Create(require: RedisFeatures.v6_2_0); + + var raw = conn.GetDatabase(); + var prefix = Me() + ":"; + var prefixed = raw.WithKeyPrefix(prefix); + raw.KeyDelete([prefix + "source", prefix + "destination"], CommandFlags.FireAndForget); + prefixed.SortedSetAdd("source", [new SortedSetEntry("a", 1), new SortedSetEntry("b", 2)], CommandFlags.FireAndForget); + + Assert.Equal(2, prefixed.SortedSetRangeAndStore("source", "destination", 0, -1)); + Assert.Equal(2, raw.SortedSetLength(prefix + "destination")); + Assert.Equal(2, raw.SortedSetLength(prefix + "source")); + + raw.KeyDelete(prefix + "destination", CommandFlags.FireAndForget); + Assert.Equal(2, await prefixed.SortedSetRangeAndStoreAsync("source", "destination", 0, -1)); + Assert.Equal(2, raw.SortedSetLength(prefix + "destination")); + } } From da99d2acdb63fcbd9eee9ceaa3bcd75e74320290 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 23 Oct 2026 16:14:59 +0100 Subject: [PATCH 20/22] Port main's fixes onto the new core by intent (merge follow-up): #3291, #3279, #3285, #3275, #3281, #3289, #3287, #3286 - #3291 (#2821) primary-only + PreferReplica: RespConnectionManager.EndpointForAny rewrites PreferReplica to DemandMaster for a primary-only command before the selector sees the flags, as Message.SetPrimaryOnly did; DemandReplica still fails. KeyTests.PrimaryOnlyCommandsWithPreferReplicaRunOnThePrimary passes. - #3279 (#3278) HELLO "server" before "version": HelloHandler reads the fields through HelloFieldMetadata; "version" is the Redis API version only when "server" is redis (absent counts as redis), a "{vendor}_version" names the product, a known non-Redis server's "version" is its own and the API version comes from INFO or the configured default, an unknown "server" is logged; the handshake applies the product variant from HELLO. ProductVariantUnitTests and ValkeyUnitTests pass. - #3285 (#3284) route as a cluster only with a slot map: RespTopology.RoutesBySlot now also wants a map, so a server that announces "cluster" while refusing CLUSTER subcommands routes as standalone (cross-slot commands go to the proxy) until a map arrives from the handshake, a refresh or a -MOVED; the refresh sweep asks SLOTS whenever the deployment may be a cluster (NeedsSlots). HELLO mode and role are published only by the interactive connection (or a reply that carried slots), so a subscription reconnect's HELLO alone cannot flip the type. RespMultiplexerExecutorTests give their topology a map where they declare a cluster; HelloClusterModeUnitTests subscribes first, since this core opens the subscription connection on first use. - #3275 follow-ups: a refused CLUSTER SLOTS is logged by the handshake, and GetEndpointsFromClusterNodes asks for the SLOTS view when only NODES is cached (ReadSlotsViewAsync), so a server that keeps refusing says so on every pass; ClusterViews is a struct, not a tuple (SanityChecks.ValueTupleNotReferenced). - #3281 sentinel demotion on a healthy connection: the heartbeat re-reads each established interactive server's role (ROLE via RespConnectionManager.TryRefreshRole), paced by ConfigCheckSeconds with the existing exponential ramp (ServerEndPoint.RoleCheckDue/OnRoleChecked), which is what the shipped bridge's CheckInfoReplication did and what RecoverDemotedPrimary needs. Also found: the new core validated commands against RawConfig.CommandMap rather than the multiplexer's effective map, so a sentinel multiplexer (CommandMap.Sentinel) refused SENTINEL when the caller's map lacked it; every RawConfig.CommandMap read in the core now uses the effective map. SentinelPrimaryDemotionUnitTests pass; SentinelTests against the real sentinels still pass. - #3289 LOADING steers a connection group away: the endpoint executor's outcome observer reports the first LOADING reply of an episode through ConnectionFailed(Loading), re-armed by a later success. Deliberately WITHOUT dropping the connection, which the shipped core did: a loading server answers everything with LOADING, its handshake included, so the drop only bought a reconnect loop, and a retrying caller then found the member mid-dial and was moved off it. LoadingRerouteTests and RetryEndToEndTests pass. - #3287 (#3283) and #3286: this core already fails connection-dropped commands with RedisConnectionException and completes a poisoned batch; ConnectionDroppedExceptionTests pins both from the public surface, replacing the two tests dropped in the merge because they drove v3 internals. - #3277 (#2894), #3273, #3293: their tests (ConnectionAttemptUnitTests, TransportCloseTests, MultiMessageWriteTimeTests) pass on this core as merged; nothing to port. Main suite net10.0: 7893 passed, 0 failed, 167 skipped. --- .../ConfigurationOptions.cs | 9 +- .../ConnectionMultiplexer.cs | 29 +++- .../Contexts/RespConnectionManager.cs | 58 +++++-- .../Contexts/RespEndpointExecutor.cs | 68 ++++++++- .../Contexts/RespHandshake.cs | 143 +++++++++++++++--- .../Contexts/RespTopology.cs | 13 +- src/StackExchange.Redis/RedisSubscriber.cs | 2 +- src/StackExchange.Redis/ServerEndPoint.cs | 25 +++ .../ConnectionDroppedExceptionTests.cs | 76 ++++++++++ .../HelloClusterModeUnitTests.cs | 4 + .../RespMultiplexerExecutorTests.cs | 22 ++- 11 files changed, 395 insertions(+), 54 deletions(-) create mode 100644 tests/StackExchange.Redis.Tests/ConnectionDroppedExceptionTests.cs diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index 9e74f461a..490046690 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -1095,9 +1095,10 @@ internal EventHandler? ConnectionAttemptCom /// default), or 0 to never do so. /// /// - /// Sends an INFO replication on each *established* interactive connection. That is how a - /// primary/replica change is noticed on a deployment that does not announce one, and it doubles as the - /// keep-alive for those sockets, which is why the interval is short. + /// Re-reads the role of each *established* interactive connection's server (ROLE; the shipped core + /// sent INFO replication). That is how a primary/replica change is noticed on a deployment that + /// does not announce one - a primary demoted in place under sentinel, say - and it doubles as a keep-alive + /// for those sockets, which is why the interval is short. /// /// Despite the name, this is not a topology re-read. It asks a server we are already talking to what it /// says about itself, so it reveals nothing about servers we cannot reach, about endpoints that have @@ -1129,7 +1130,7 @@ public int ConfigCheckSeconds /// /// /// Distinct from , which despite its name does not re-read topology - - /// it sends an INFO replication on an established connection to check a server's role. + /// it asks an established connection's server for its role. /// /// public int TopologyRefreshSeconds diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index a0e900bbe..e6053b743 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -2282,25 +2282,32 @@ private static async Task ReadClusterViewsAsync(ServerEndPoint ser var slotsRead = ReadClusterSlotsAsync(server).ObserveErrors(); var nodesRead = new RedisServer(server, null).ClusterNodesAsync().ObserveErrors(); - ClusterTopology? topology; + var topology = await AwaitSlotsViewAsync(slotsRead, server, log).ForAwait(); try { - topology = ClusterTopology.From(await slotsRead.ForAwait()); + return new(await nodesRead.ForAwait(), topology); } catch (RedisServerException ex) { - log?.LogInformationClusterSlotsRefused(new(server), ex.Message); - topology = null; + log?.LogInformationClusterNodesRefused(new(server), ex.Message); + return new(null, topology); } + } + + /// The SLOTS view alone, or null when the server refused it (logged, not thrown). + private static Task ReadSlotsViewAsync(ServerEndPoint server, ILogger? log) + => AwaitSlotsViewAsync(ReadClusterSlotsAsync(server), server, log); + private static async Task AwaitSlotsViewAsync(Task slotsRead, ServerEndPoint server, ILogger? log) + { try { - return new(await nodesRead.ForAwait(), topology); + return ClusterTopology.From(await slotsRead.ForAwait()); } catch (RedisServerException ex) { - log?.LogInformationClusterNodesRefused(new(server), ex.Message); - return new(null, topology); + log?.LogInformationClusterSlotsRefused(new(server), ex.Message); + return null; } } @@ -2372,6 +2379,14 @@ private readonly struct ClusterViews(ClusterConfiguration? nodes, ClusterTopolog clusterConfig = asked.Nodes; topology ??= asked.Topology; } + else if (topology is null) + { + // NODES is known but nobody holds a SLOTS view: ask for one, so that a server which keeps + // refusing it says so in the log on every pass (main #3275) rather than leaving "why is + // there no slot map?" unanswered, and so that a cluster whose refusal was transient gets + // its map back without waiting for a redirect + topology = await ReadSlotsViewAsync(server, log).ForAwait(); + } } else { diff --git a/src/StackExchange.Redis/Contexts/RespConnectionManager.cs b/src/StackExchange.Redis/Contexts/RespConnectionManager.cs index f344e7301..55418893d 100644 --- a/src/StackExchange.Redis/Contexts/RespConnectionManager.cs +++ b/src/StackExchange.Redis/Contexts/RespConnectionManager.cs @@ -111,7 +111,7 @@ internal RespConnectionManager(ConnectionMultiplexer multiplexer) RoleLearned = PublishRoleToSelector, }; _features = new ObservedFeatureProbe(this); - _select = new SelectPreamble(new RespContext(multiplexer.RawConfig.CommandMap)); + _select = new SelectPreamble(new RespContext(multiplexer.CommandMap)); _defaultDatabase = multiplexer.RawConfig.DefaultDatabase.GetValueOrDefault(); _router = Rebind(multiplexer.RawConfig.DefaultDatabase.GetValueOrDefault()); SeedRoles(); @@ -186,7 +186,7 @@ internal int ConnectedEndpointCountForTest internal RespDatabaseContext GetDatabase(int database = 0) { return new(new RespContext( - _multiplexer.RawConfig.CommandMap, + _multiplexer.CommandMap, database: database, serverType: _multiplexer.ServerSelectionStrategy.ServerType) .WithTopology(_topology) @@ -469,6 +469,15 @@ private static bool Faulted( if (wantsReplica && PickReplica(_topology.Replicas) is { } replica) return replica; + // A primary-only command sent with PreferReplica goes to the primary, which the shipped core + // arranged by rewriting the flags before the selector saw them (Message.SetPrimaryOnly; main #3291, + // #2821): the selector honours the preference as written, and the pre-send check would then refuse + // the command on the replica it chose. DemandReplica is left alone so that it still fails the same way. + if (command.IsPrimaryOnly() && CommandFlagsInternal.GetPrimaryReplicaFlags(flags) == CommandFlags.PreferReplica) + { + flags = CommandFlagsInternal.SetPrimaryReplicaFlags(flags, CommandFlags.DemandMaster); + } + // ...and the selector for the keyless choice, for the same reason and with the same measurement: // it knows every configured endpoint's role and selectability without this core having dialled // any of them. The role-aware choice below is what will replace it. @@ -758,7 +767,7 @@ internal RespEndpointExecutor SubscriptionEndpoint(EndPoint endpoint) /// internal RespServerContext ServerContext(EndPoint endpoint) => new(new RespContext( - _multiplexer.RawConfig.CommandMap, + _multiplexer.CommandMap, database: -1, serverType: _multiplexer.ServerSelectionStrategy.ServerType) .WithScriptCache(_multiplexer.ScriptCache) @@ -776,7 +785,7 @@ internal RespServerContext ServerContext(EndPoint endpoint) /// prefix, because that IS the channel as far as the wire is concerned. /// internal RespContext SubscriptionContext(EndPoint endpoint) - => new RespContext(_multiplexer.RawConfig.CommandMap) + => new RespContext(_multiplexer.CommandMap) .AppendChannelPrefix(_multiplexer.RawConfig.ChannelPrefix) .WithServices(_features) .WithExecutor(SubscriptionEndpoint(endpoint)); @@ -1209,7 +1218,7 @@ await RespHandshake.DiscoverServerConfigAsync( && result.Protocol < RedisProtocol.Resp3 && config.TryResp3() && _multiplexer.RawConfig.SharedSubscriptionConnection - && _multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE) + && _multiplexer.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE) && (_multiplexer.AnySubscribed() || _multiplexer.ConfigurationChangedChannel is not null)) { @@ -1331,7 +1340,7 @@ await SubscribeToConfigurationChannelAsync(context, cancellationToken) internal void DialSubscriptionSocketForConfigurationChannel(EndPoint endpoint) { if (_multiplexer.ConfigurationChangedChannel is null - || !_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE)) + || !_multiplexer.CommandMap.IsAvailable(RedisCommand.SUBSCRIBE)) { return; } @@ -2121,7 +2130,7 @@ internal void AddCounters(EndPoint endpoint, ConnectionType connectionType, Conn internal void SetLibraryName(string libraryName) { if (string.IsNullOrWhiteSpace(libraryName)) return; - if (!_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.CLIENT)) return; + if (!_multiplexer.CommandMap.IsAvailable(RedisCommand.CLIENT)) return; foreach (var pair in _endpoints) Announce(pair.Key, pair.Value); foreach (var pair in _subscriptions) Announce(pair.Key, pair.Value); @@ -2131,7 +2140,7 @@ void Announce(EndPoint endpoint, RespEndpointExecutor executor) if (!executor.IsConnectedNow) return; try { - var context = new RespContext(_multiplexer.RawConfig.CommandMap, database: -1) + var context = new RespContext(_multiplexer.CommandMap, database: -1) .WithExecutor(executor.WithDatabase(-1)); _ = context.SendAsync( $"{RedisCommand.CLIENT}{RedisLiterals.SETINFO}{RedisLiterals.lib_name}{(RedisValue)libraryName}", @@ -2170,6 +2179,7 @@ private void Publish(ServerEndPoint server, in RespHandshakeResult result) { if (result.Version is { } version) server.Version = version; if (result.RoleFromHello is { } isReplica) server.IsReplica = isReplica; + server.OnRoleChecked(); // the handshake read the role; the heartbeat's re-read starts counting from here // The SLOTS view first, and this ordering is load-bearing: SetClusterConfiguration below reads // ServerEndPoint.ClusterTopology to decide which reply drives the selector's slot map, and falls @@ -2255,7 +2265,7 @@ internal void OnRole(EndPoint endpoint, bool isReplica) /// internal bool TryRefreshRole(EndPoint endpoint, ILogger? log = null) { - if (!_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.ROLE) + if (!_multiplexer.CommandMap.IsAvailable(RedisCommand.ROLE) || !_endpoints.TryGetValue(endpoint, out var executor) || !executor.IsConnectedNow) { @@ -2269,7 +2279,7 @@ async Task RefreshAsync() { try { - var context = new RespDatabaseContext(new RespContext(_multiplexer.RawConfig.CommandMap, database: -1).WithExecutor(executor)); + var context = new RespDatabaseContext(new RespContext(_multiplexer.CommandMap, database: -1).WithExecutor(executor)); var role = await context.SendAsync($"{RedisCommand.ROLE}", handler: RespHandshake.RoleHandler.Instance).ConfigureAwait(false); _topology.OnRole(endpoint, role.Role); if (role.Peers is not null) @@ -2357,9 +2367,11 @@ private void PublishRoleToSelector(EndPoint endpoint, bool isReplica) /// internal async Task RefreshTopologyAsync(ILogger? log = null) { - if (_topology.RoutesBySlot) + // NeedsSlots rather than RoutesBySlot: a cluster that has refused SLOTS so far routes as standalone + // (RoutesBySlot wants a map), and this sweep is exactly how it would ever get one + if (_topology.NeedsSlots) { - if (!_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.CLUSTER)) return; + if (!_multiplexer.CommandMap.IsAvailable(RedisCommand.CLUSTER)) return; foreach (var pair in _endpoints) { @@ -2387,7 +2399,7 @@ internal async Task RefreshTopologyAsync(ILogger? log = null) return; } - if (!_topology.WantsRoles || !_multiplexer.RawConfig.CommandMap.IsAvailable(RedisCommand.ROLE)) return; + if (!_topology.WantsRoles || !_multiplexer.CommandMap.IsAvailable(RedisCommand.ROLE)) return; foreach (var pair in _endpoints) { @@ -2411,7 +2423,7 @@ internal async Task RefreshTopologyAsync(ILogger? log = null) } RespDatabaseContext Context(RespEndpointExecutor executor) - => new(new RespContext(_multiplexer.RawConfig.CommandMap, database: -1).WithExecutor(executor)); + => new(new RespContext(_multiplexer.CommandMap, database: -1).WithExecutor(executor)); // this core's map AND the selector's, for the same reason the handshake does both: the selector // still answers for IServer and for the routing fallbacks, and a refresh that corrected only one @@ -2557,6 +2569,24 @@ internal void OnHeartbeat() var timeout = _multiplexer.AsyncTimeoutMilliseconds; foreach (var pair in _endpoints) pair.Value.OnHeartbeat(timeout); foreach (var pair in _subscriptions) pair.Value.OnHeartbeat(timeout); + + // The periodic role re-read the shipped bridge made on its heartbeat (CheckInfoReplication, paced by + // ConfigCheckSeconds with the exponential ramp a suspected failover forces). A node demoted in place + // reports its new role only to somebody who asks, and nothing else here would: the sentinel recovery + // path (main #3281) acts on ServerEndPoint.IsReplica, which this is what keeps honest. Interactive + // endpoints only, as before, and never a sentinel, which has no role worth asking for. + foreach (var pair in _endpoints) + { + if (!pair.Value.IsConnectedNow) continue; + if (_multiplexer.TryResolveServerEndPoint(pair.Key) is not { } server + || server.ServerType == ServerType.Sentinel + || !server.RoleCheckDue) + { + continue; + } + + if (TryRefreshRole(pair.Key, _multiplexer.Logger)) server.OnRoleChecked(); + } } /// Release a connection-local claim on this core's interactive connection to an endpoint. diff --git a/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs b/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs index a1dd90659..3a84ad4ae 100644 --- a/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs +++ b/src/StackExchange.Redis/Contexts/RespEndpointExecutor.cs @@ -1337,6 +1337,45 @@ private bool Enqueue(RespPayloadOperation operation) /// Strictly healthy -> tripped -> actuated, then healthy again with fresh counters. private int _circuitBreakerState; + /// Whether the current LOADING episode has been reported; cleared by a success, and per connection. + private int _loadingReported; + + /// The LOADING reply being reported, for . + private RedisServerException? _loadingFault; + + private static readonly WaitCallback s_LoadingObserved = static state => + { + var executor = (RespEndpointExecutor)state!; + try + { + if (Volatile.Read(ref executor._loadingFault) is { } loading) executor.ActuateLoading(loading); + } + catch + { + // a report that cannot be made is not a reason to bring the process down with it + } + }; + + /// Report a LOADING reply through ConnectionFailed, as a breaker trip is reported. + /// + /// From the pool rather than the completion path, as is, so that the thread that + /// just finished somebody's command does not run the group's handlers. reconfigure: false: a + /// loading server has not changed the topology, and a group's failover is driven by the event itself. + /// + private void ActuateLoading(RedisServerException loading) + { + if (_endpoint is { } reportAt && Server is { } server && !server.Multiplexer.IsDisposed) + { + server.Multiplexer.OnConnectionFailed( + reportAt, + IsSubscriptionEndpoint ? ConnectionType.Subscription : ConnectionType.Interactive, + ConnectionFailureType.Loading, + loading, + reconfigure: false, + physicalName: null); + } + } + private const int CircuitHealthy = 0, CircuitTripped = 1, CircuitActuated = 2; /// @@ -1349,6 +1388,30 @@ private bool Enqueue(RespPayloadOperation operation) /// void IRespOutcomeObserver.ObserveOutcome(Exception? fault) { + // A server that is loading its dataset is a server that cannot serve. The shipped core treated the + // first LOADING reply as a connection failure (RecordConnectionFailed(ConnectionFailureType.Loading), + // main #3289): ConnectionFailed is the event a connection group listens for to steer away, and the + // breaker cannot be what moves it, since a loading server answers everything - its handshake included + // - with the same error. The event is kept; the drop is not. The shipped core tore the connection down + // and dialled again into a handshake that failed the same way, and a retrying caller then found the + // member mid-dial and was moved off it (RetryEndToEndTests.WithRetry_WhenFailoverNeverArrives...). + // A LOADING reply is prompt, and a prompt refusal is a better answer than a reconnect loop. Said once + // per episode: the burst that follows the first reply is the same fact repeated, and a success in + // between re-arms it, since a replica re-syncing with replica-serve-stale-data off can go LOADING + // again on the same long-lived connection. + if (fault is RedisServerException { Kind: RedisErrorKind.Loading } loading) + { + if (Interlocked.CompareExchange(ref _loadingReported, 1, 0) == 0) + { + Volatile.Write(ref _loadingFault, loading); // carried in a field: the library does not reference ValueTuple + ThreadPool.QueueUserWorkItem(s_LoadingObserved, this); + } + } + else if (fault is null && Volatile.Read(ref _loadingReported) != 0) + { + Volatile.Write(ref _loadingReported, 0); + } + var accumulator = Volatile.Read(ref _circuitBreaker); if (accumulator is null || !accumulator.Trip(fault)) return; @@ -1399,6 +1462,7 @@ private void ActuateTrip() Volatile.Write(ref _circuitBreaker, _circuitBreakerFactory?.Invoke()); Volatile.Write(ref _circuitBreakerState, CircuitHealthy); + Volatile.Write(ref _loadingReported, 0); } /// Bring this endpoint up now, and wait for it. @@ -1753,7 +1817,7 @@ private async Task PingAsync() { var started = DateTime.UtcNow; await new RespDatabaseContext( - new RespContext(Server!.Multiplexer.RawConfig.CommandMap, database: -1).WithExecutor(this)) + new RespContext(Server!.Multiplexer.CommandMap, database: -1).WithExecutor(this)) .PingAsync(CommandFlags.NoRedirect) .ConfigureAwait(false); @@ -1900,7 +1964,7 @@ private void Validate(in RespRequest request, int database) // core that renders its own frames has to make it somewhere; found by porting SUBSCRIBE, which // is what ConfigTests.ConnectWithSubscribeDisabled asks about. if (request.Command is not (RedisCommand.NONE or RedisCommand.UNKNOWN) - && !config.CommandMap.IsAvailable(request.Command)) + && !server.Multiplexer.CommandMap.IsAvailable(request.Command)) { throw ExceptionFactory.CommandDisabled(request.Command); } diff --git a/src/StackExchange.Redis/Contexts/RespHandshake.cs b/src/StackExchange.Redis/Contexts/RespHandshake.cs index 7c11a5049..25ddaf083 100644 --- a/src/StackExchange.Redis/Contexts/RespHandshake.cs +++ b/src/StackExchange.Redis/Contexts/RespHandshake.cs @@ -306,6 +306,19 @@ internal static async Task PerformAsync( version = hello.Version; roleFromHello = hello.IsReplica; + // what HELLO said the product is, when it named a vendor (main #3279): "server" for a server that + // is not Redis, or a "{vendor}_version" field beside a Redis-compatible "version" + if (server is not null) + { + if (hello.Variant is { } variant && hello.ProductVersion is { Length: > 0 } productVersion) + { + server.SetProductVariant(variant, productVersion); + log?.LogInformationAutoConfiguredHelloProductVariant(new(server), variant, productVersion); + } + + if (hello.UnknownServer is { } unknown) log?.LogInformationAutoConfiguredHelloUnknownServer(new(server), unknown); + } + if (log is not null) { log.LogInformationAutoConfiguredHelloProtocol(new(server!), protocol); @@ -402,9 +415,11 @@ internal static async Task PerformAsync( // slots exist, so this is a cluster whatever CLUSTER INFO did or did not say if (slots is { Assignments.Count: > 0 }) serverType = ServerType.Cluster; } - catch (RedisServerException) + catch (RedisServerException ex) { - // no map from this server; the selector still answers + // no map from this server; the selector still answers. Said in the log (main #3275), because + // the consequence - a cluster routed as standalone until a map arrives - is otherwise silent + if (server is not null) log?.LogInformationClusterSlotsRefused(new(server), ex.Message); } } @@ -416,8 +431,16 @@ internal static async Task PerformAsync( // BEFORE the connection is handed back, and that ordering is the point: the endpoint executor // publishes the connection and drains its backlog the moment this returns, and a backlog - // draining against an unset topology is exactly the window that loses per-slot ordering - topology?.OnServerType(serverType); + // draining against an unset topology is exactly the window that loses per-slot ordering. + // + // ...and only from the connection that does discovery (main #3285, #3284). The subscription + // connection's handshake is HELLO alone, with no INFO or CLUSTER follow-up, so a proxy whose HELLO + // disagrees with its INFO (AMR says "cluster" to one and "standalone" to the other) would otherwise + // have the last word every time pub/sub reconnects on its own. "mode" and "role" describe the + // server, and the interactive connection is the one that establishes them; a reply that carried + // slots is proof of a cluster whichever socket asked, and counts. + var discovers = connectionType == ConnectionType.Interactive || slots is { Assignments.Count: > 0 }; + if (discovers) topology?.OnServerType(serverType); if (slots is not null && topology is not null) { @@ -573,7 +596,8 @@ internal static async Task PerformAsync( LogResponse(log, server, connectionType, "SELECT " + database.ToString(CultureInfo.InvariantCulture), "OK"); } - return new RespHandshakeResult(protocol, serverType, version, knowServerType, connectionId, roleFromHello, clusterNodes, slots); + // the server type and role are published only by the connection that discovers them (see `discovers`) + return new RespHandshakeResult(protocol, serverType, version, knowServerType && discovers, connectionId, discovers ? roleFromHello : null, clusterNodes, slots); } /// Read the server-wide settings the client models, for a server nothing has described yet. @@ -1057,9 +1081,12 @@ private static async Task DiscoverTieBreakerAsync(RespDatabaseContext context, S /// never learns the product applies the wrong rule to a real deployment. /// /// - /// From INFO, not from HELLO, which is deliberate and matches the v3 core. - /// HELLO does carry a server field, and reading it would be free - but it names the - /// product without its product version, and the version is half of what is being established. The + /// From INFO, unless HELLO already named the product. HELLO carries a + /// server field and, for some products, a {vendor}_version beside it; since main #3279 the + /// handshake reads those (see HelloHandler), and a server so described skips this. What is left + /// for INFO is a server whose HELLO said nothing - RESP2-only, or plain "redis" with no + /// vendor field - and, separately, the Redis API version of a Valkey, whose HELLO "version" is its + /// own (that is the version fallback above, not this). The /// variant is decided across the whole reply rather than per line, because both /// redis_version and valkey_version can be present in either order; that was the /// v3 processor's rule and this reuses its field table rather than growing a second copy. @@ -1373,16 +1400,36 @@ await context.SendAsync($"{command}{RedisLiterals.TRACKING}{RedisLiterals.ON}{Re /// The version it reported, if it did. /// What it said this server's role is, when it said. /// What it said this connection's id is, when it said. + /// The product the reply named, when it named one other than plain Redis. + /// That product's own version, when is set. + /// A "server" value this library does not know, for logging; null otherwise. /// /// A struct rather than a tuple: the library must not reference System.ValueTuple, which /// would add a facade dependency on the down-level targets - asserted by /// SanityCheckTests.ValueTupleNotReferenced, which is how this was caught. /// - private readonly struct HelloReply(int proto, ServerType? mode, Version? version, bool? isReplica, long? connectionId = null) + private readonly struct HelloReply( + int proto, + ServerType? mode, + Version? version, + bool? isReplica, + long? connectionId = null, + ProductVariant? variant = null, + string? productVersion = null, + string? unknownServer = null) { /// What the reply said this connection's id is, when it said. internal long? ConnectionId { get; } = connectionId; + /// The product the reply named, when it named one other than plain Redis. + internal ProductVariant? Variant { get; } = variant; + + /// That product's own version, when is set. + internal string? ProductVersion { get; } = productVersion; + + /// A "server" value this library does not know, so that it can be logged; null otherwise. + internal string? UnknownServer { get; } = unknownServer; + internal int Proto { get; } = proto; internal ServerType? Mode { get; } = mode; @@ -1444,35 +1491,66 @@ public HelloReply Parse(ref RespReader reader) { var proto = 2; // a reply we could not read is not a reason to claim RESP3 ServerType? mode = null; - Version? version = null; bool? isReplica = null; long? connectionId = null; + // "version" means different things depending on "server" (main #3279, #3278): Redis - and Garnet, + // which presents as "redis" and adds "garnet_version" - report the Redis API version, but Valkey + // reports ITS OWN ("server" => "valkey", "version" => "9.1.2", while INFO still says + // redis_version:7.2.4). So what "version" is can only be decided once the whole map has been read. + var serverIsRedis = true; // absent means redis: older servers + ProductVariant? helloVariant = null, vendorVariant = null; + string? versionText = null, vendorVersion = null, unknownServer = null; + var count = reader.AggregateLength(); for (var i = 0; i < count; i++) { if (!reader.TryMoveNext()) break; - var isProto = reader.Is("proto"u8); - var isMode = !isProto && reader.Is("mode"u8); - var isVersion = !isProto && !isMode && reader.Is("version"u8); - var isRole = !isProto && !isMode && !isVersion && reader.Is("role"u8); - var isId = !isProto && !isMode && !isVersion && !isRole && reader.Is("id"u8); + var field = reader.IsScalar && reader.TryGetSpan(out var name) && HelloFieldMetadata.TryParse(name, out var parsed) + ? parsed + : HelloField.Unknown; if (!reader.TryMoveNext()) break; i++; - if (isProto && reader.IsScalar && reader.TryReadInt64(out var value)) + if (field == HelloField.Proto && reader.IsScalar && reader.TryReadInt64(out var value)) { proto = (int)value; } - else if (isId && reader.IsScalar && reader.TryReadInt64(out var id)) + else if (field == HelloField.Id && reader.IsScalar && reader.TryReadInt64(out var id)) { connectionId = id; } - else if (isVersion && reader.IsScalar) + else if (field == HelloField.Server && reader.IsScalar) { - _ = Format.TryParseVersion(reader.ReadString(), out version); + if (reader.TryGetSpan(out var span) && ProductVariantMetadata.TryParse(span, out var named)) + { + helloVariant = named; + serverIsRedis = named == ProductVariant.Redis; + } + else + { + serverIsRedis = false; + unknownServer = reader.ReadString() ?? ""; + } + } + else if (field == HelloField.Version && reader.IsScalar) + { + versionText = reader.ReadString(); } - else if (isRole && reader.IsScalar) + else if (field is HelloField.GarnetVersion or HelloField.ValkeyVersion or HelloField.DragonflyVersion + or HelloField.MemuraiVersion or HelloField.RedictVersion && reader.IsScalar) + { + vendorVariant = field switch + { + HelloField.GarnetVersion => ProductVariant.Garnet, + HelloField.ValkeyVersion => ProductVariant.Valkey, + HelloField.DragonflyVersion => ProductVariant.Dragonfly, + HelloField.MemuraiVersion => ProductVariant.Memurai, + _ => ProductVariant.Redict, + }; + vendorVersion = reader.ReadString(); + } + else if (field == HelloField.Role && reader.IsScalar) { // FREE, and it removes the need to ask: the v3 handshake tracked exactly this // as RoleKnownFromHello, and skipped its SET probe when it was set. #2968 is about @@ -1483,7 +1561,7 @@ public HelloReply Parse(ref RespReader reader) isReplica = replica; } } - else if (isMode && reader.IsScalar) + else if (field == HelloField.Mode && reader.IsScalar) { // "standalone" | "sentinel" | "cluster"; only the last changes routing mode = reader.Is("cluster"u8) ? ServerType.Cluster @@ -1496,7 +1574,28 @@ public HelloReply Parse(ref RespReader reader) } } - return new HelloReply(proto, mode, version, isReplica, connectionId); + Version? version = null; + ProductVariant? variant = null; + string? productVersion = null; + if (serverIsRedis) + { + // "version" is the Redis API version; a "{vendor}_version" field, if any, names the product + if (versionText is not null) _ = Format.TryParseVersion(versionText, out version); + if (vendorVariant is { } fromField && !string.IsNullOrWhiteSpace(vendorVersion)) + { + variant = fromField; + productVersion = vendorVersion; + } + } + else if (helloVariant is { } fromServer) + { + // a known non-Redis server: "version" is the vendor's own, and the Redis API version has to + // come from INFO (redis_version), or stay at the configured default - never from here + variant = fromServer; + productVersion = !string.IsNullOrWhiteSpace(vendorVersion) ? vendorVersion : versionText; + } + + return new HelloReply(proto, mode, version, isReplica, connectionId, variant, productVersion, unknownServer); } } diff --git a/src/StackExchange.Redis/Contexts/RespTopology.cs b/src/StackExchange.Redis/Contexts/RespTopology.cs index 55314988e..40990fbba 100644 --- a/src/StackExchange.Redis/Contexts/RespTopology.cs +++ b/src/StackExchange.Redis/Contexts/RespTopology.cs @@ -162,7 +162,18 @@ internal void OnServerType(ServerType serverType) /// Whether the slot decides which endpoint serves the command. /// False while unknown: routing on a guess is wrong, not merely wasteful. - internal bool RoutesBySlot => (RespClusterState)Volatile.Read(ref _state) == RespClusterState.Yes; + /// Whether commands are routed by slot: a cluster, with a slot map. + /// + /// A server saying it is a cluster is not enough (main #3285, #3284): proxied deployments announce + /// "cluster" in HELLO while refusing every CLUSTER subcommand, and with no map the only + /// thing cluster routing would add is refusing cross-slot commands client-side that the proxy would have + /// accepted. So until a map exists - from the handshake, a refresh, or a -MOVED - routing falls to + /// the selector as it does for a standalone server; the map is never cleared once populated, so an + /// established cluster whose refresh is refused keeps routing as one. is the + /// other question - "might this be a cluster?" - and stays true here so slots are still computed and + /// asked for. + /// + internal bool RoutesBySlot => (RespClusterState)Volatile.Read(ref _state) == RespClusterState.Yes && Volatile.Read(ref _slots) is not null; /// The server type, as far as anyone knows. internal ServerType ServerType => RoutesBySlot ? ServerType.Cluster : ServerType.Standalone; diff --git a/src/StackExchange.Redis/RedisSubscriber.cs b/src/StackExchange.Redis/RedisSubscriber.cs index d817d0e84..130b6100c 100644 --- a/src/StackExchange.Redis/RedisSubscriber.cs +++ b/src/StackExchange.Redis/RedisSubscriber.cs @@ -348,7 +348,7 @@ private Task PingConnectionAsync(CommandFlags flags) // cores, handing back null sent the ping down the v3 path, which built a bridge (and dialled // it) just to answer. The round trip that remains meaningful is the one on the ordinary // connection to that server. - var direct = new RespContext(multiplexer.RawConfig.CommandMap, database: -1) + var direct = new RespContext(multiplexer.CommandMap, database: -1) .WithExecutor(multiplexer.Connections.ServerExecutor(server.EndPoint)); return Measured(new RespDatabaseContext(direct).PingMeasureAsync(flags)); } diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index f1acdda17..9d5a59459 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -812,6 +812,31 @@ public EndPoint? PrimaryEndPoint [ThreadStatic] private static Random? r; + private int lastRoleCheckTicks; + + /// Seconds since this server's role was last read, by the handshake or by the heartbeat. + internal int LastRoleCheckSecondsAgo => unchecked(Environment.TickCount - Volatile.Read(ref lastRoleCheckTicks)) / 1000; + + /// + /// Whether the heartbeat should re-read this server's role now: have passed + /// since the last read. Zero disables the check, as it did for the shipped core's INFO replication. + /// + internal bool RoleCheckDue => ConfigCheckSeconds > 0 && LastRoleCheckSecondsAgo >= ConfigCheckSeconds; + + /// + /// Note that the role was just read, and ease the pacing back towards the configured interval. + /// + /// + /// What CheckInfoReplication recorded in the shipped core before sending its INFO replication; + /// here the read itself is RespConnectionManager.TryRefreshRole, driven from the core's heartbeat, + /// and the handshake counts as a read too, since it learns the role as it connects. + /// + internal void OnRoleChecked() + { + Volatile.Write(ref lastRoleCheckTicks, Environment.TickCount); + ResetExponentiallyReplicationCheck(); + } + /// /// Forces frequent replication check starting from 1 second up to max ConfigCheckSeconds with an exponential increment. /// diff --git a/tests/StackExchange.Redis.Tests/ConnectionDroppedExceptionTests.cs b/tests/StackExchange.Redis.Tests/ConnectionDroppedExceptionTests.cs new file mode 100644 index 000000000..1d4cf9a71 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ConnectionDroppedExceptionTests.cs @@ -0,0 +1,76 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// A command that fails because its connection is gone must fail with a : +/// never the raw transport exception, and never a when no caller token was +/// involved (#3283, main #3287) - and a batch whose connection is gone must complete every one of its commands +/// that way rather than leaving any of them pending (#3286). The shipped core's tests drove its bridge and +/// physical connection directly; this core is driven from the public surface, where the same facts are +/// observable. +/// +public class ConnectionDroppedExceptionTests(ITestOutputHelper log) +{ + private static async Task AwaitFailure(Task task) + { + var completed = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(10))); + Assert.Same(task, completed); + return await Assert.ThrowsAnyAsync(() => task); + } + + private async Task<(InProcessTestServer Server, ConnectionMultiplexer Connection)> ConnectThenLoseTheServerAsync() + { + var server = new InProcessTestServer(log); + var config = server.GetClientConfig(withPubSub: false, defaultOnly: true); + config.BacklogPolicy = BacklogPolicy.FailFast; + config.AbortOnConnectFail = false; + config.AllowSimulateConnectionFailure = true; + var conn = await ConnectionMultiplexer.ConnectAsync(config); + + // no connection, and no way to make one: what a command meets when the server has gone away + conn.AllowConnect = false; + conn.GetServerSnapshot()[0].SimulateConnectionFailure(SimulatedFailureType.All); + Assert.False(conn.IsConnected); + return (server, conn); + } + + [Fact] + public async Task ACommandWithNoConnectionFailsWithRedisConnectionExceptionNotCancellation() + { + var (server, conn) = await ConnectThenLoseTheServerAsync(); + using var _ = server; + await using var __ = conn; + + var task = conn.GetDatabase().PingAsync(); + var ex = await AwaitFailure(task); + log.WriteLine(ex.ToString()); + Assert.IsType(ex); + Assert.False(task.IsCanceled); + } + + [Fact] + public async Task ABatchWithNoConnectionCompletesEveryCommandWithRedisConnectionException() + { + var (server, conn) = await ConnectThenLoseTheServerAsync(); + using var _ = server; + await using var __ = conn; + + var batch = conn.GetDatabase().CreateBatch(); + var tasks = Enumerable.Range(0, 20).Select(i => batch.StringGetAsync("k" + i)).ToArray(); + batch.Execute(); + + // every command, not most: a batch's tail used to be left pending when its connection was poisoned + var all = Task.WhenAll(tasks); + var completed = await Task.WhenAny(all, Task.Delay(TimeSpan.FromSeconds(10))); + Assert.Same(all, completed); + foreach (var task in tasks) + { + Assert.True(task.IsFaulted, "every command must fault, not cancel or hang"); + Assert.IsType(task.Exception!.InnerException); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs index 50336b9ab..150fabe8f 100644 --- a/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/HelloClusterModeUnitTests.cs @@ -46,6 +46,10 @@ public async Task CrossSlotSurvivesSubscriptionReconnect(RedisProtocol protocol) await db.StringGetAsync(crossSlot); // sanity: fine on a fresh connection Assert.Equal(ServerType.Standalone, conn.ServerSelectionStrategy.ServerType); + // the new core opens the subscription connection on first use rather than at connect, so give it a + // reason to exist: without one there is nothing to drop below, and the test would time out waiting + // for a reconnect that never has to happen + await conn.GetSubscriber().SubscribeAsync(RedisChannel.Literal("hello-mode"), static (_, _) => { }); var endpoint = conn.GetServerSnapshot()[0]; await WaitUntilAsync(() => endpoint.IsSubscriberConnected, "subscriber connected"); diff --git a/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs b/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs index ff746c0f6..b90c926f1 100644 --- a/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs +++ b/tests/StackExchange.Redis.Tests/RespMultiplexerExecutorTests.cs @@ -49,10 +49,26 @@ private sealed class Router } } + /// + /// What discovery tells the topology when it finds a cluster: the server type and a slot map. + /// + /// + /// Routing by slot needs both (, main #3285): a server that merely + /// says "cluster" while giving no map is routed as standalone, so these tests, whose + /// answers the slot question itself, give the topology a map covering every slot to stand for "SLOTS was + /// answered". The owner recorded there is never consulted - the router is. + /// + private static void LearnCluster(RespTopology topology) + { + topology.OnServerType(ServerType.Cluster); + topology.SetSlotRange(0, ServerSelectionStrategy.TotalSlots - 1, new System.Net.IPEndPoint(System.Net.IPAddress.Loopback, 6379)); + } + private static (RespDatabaseContext Context, RespTopology Topology, Router Router) Build( RespClusterState known = RespClusterState.No) { var topology = new RespTopology(known); + if (known == RespClusterState.Yes) LearnCluster(topology); // a known cluster comes with its map var router = new Router { Default = new Endpoint("default", "$2\r\nok\r\n") }; var executor = new RespMultiplexerExecutor(topology, router.ForSlot, router.Any); var raw = new RespContext(serverType: ServerType.Standalone).WithTopology(topology).WithExecutor(executor); @@ -102,7 +118,7 @@ public async Task LearningItIsAClusterChangesRoutingWithoutRebuildingAnything() Assert.True(router.Default!.HasSent); // went to the only endpoint we knew about Assert.False(owner.HasSent); - topology.OnServerType(ServerType.Cluster); // the discovery + LearnCluster(topology); // the discovery Assert.Equal("mine", (string?)await context.Strings.GetAsync("user:1")); Assert.True(owner.HasSent); // and now it is routed by slot @@ -140,7 +156,7 @@ public async Task ARequestRenderedBeforeTheAnswerArrivedIsStillRoutable() using var request = context.Raw.Render($"{RedisCommand.GET}{(RedisKey)"user:1"}").Detach(); Assert.Equal(slot, request.Slot); // rendered while nobody knew, and carries its slot - topology.OnServerType(ServerType.Cluster); // ...and now we know + LearnCluster(topology); // ...and now we know var executor = context.Raw.Executor!; using var payload = await executor.SendAsync(request); @@ -217,7 +233,7 @@ public void RoutingQuestionsFollowTheSameTopology() Assert.True(db.IsConnected("user:1")); Assert.Equal(0, router.SlotLookups); // standalone: no hashing even for the routing question - topology.OnServerType(ServerType.Cluster); + LearnCluster(topology); Assert.True(db.IsConnected("user:1")); Assert.Equal(1, router.SlotLookups); // cluster: now it asks who owns the slot } From 38014f3a59e3379d0762cdf0bad340791a666da3 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 23 Oct 2026 16:36:40 +0100 Subject: [PATCH 21/22] StyleCop.Analyzers 1.3.151-beta on v4 (main #3296 reached the new core): the 79 sites the shared tidy could not see Held at 1.2.0-beta.556 through the main merge so that the merge and the intent ports could be judged on their own; this takes main's pin and answers what 1.3 says about v4-only code. All 79 in src/StackExchange.Redis: - SA1653 (54): partial types documented in every part. The primary file keeps its ; the other parts (RedisDatabase.*.cs, the Groups *.Methods/*.Types/*.Sort parts, RespClientCache.Slabs.cs) use , as main's tidy did for its own partials. - SA1143 (11): ternaries laid out with ? and : each starting a line; two multi-line conditions (PFCOUNT's replica safety, XREADGROUP's read-only category) moved into a named local first. - SA1504 (8): a single-line getter beside a multi-line setter/init made block-bodied. - SA1315 (5): IInternalConnectionMultiplexer implementations take the interface's parameter names (endPoint, type). - SA1407 (1): parenthesised the comparison beside a non-short-circuit &. No behavioural change. Main suite net10.0: 7893 passed, 0 failed, 167 skipped. --- Directory.Packages.props | 2 +- .../Caching/CacheOptions.cs | 30 +++++++++++++++---- .../Caching/RespClientCache.Slabs.cs | 4 +-- .../ConnectionMultiplexer.FeatureFlags.cs | 8 ++--- .../ConnectionMultiplexer.cs | 7 +++-- .../Contexts/RespConnectionManager.cs | 8 +++-- .../Contexts/RespHandshake.cs | 8 +++-- .../Contexts/RespPayloadOperation.cs | 12 ++++++-- .../Database/RedisDatabase.Arrays.cs | 4 +-- .../Database/RedisDatabase.Bitmaps.cs | 4 +-- .../Database/RedisDatabase.Execute.cs | 4 +-- .../Database/RedisDatabase.Geospatial.cs | 4 +-- .../Database/RedisDatabase.Hashes.cs | 4 +-- .../Database/RedisDatabase.HyperLogLog.cs | 4 +-- .../Database/RedisDatabase.Keys.cs | 4 +-- .../Database/RedisDatabase.Lists.cs | 4 +-- .../Database/RedisDatabase.PubSub.cs | 4 +-- .../Database/RedisDatabase.Scans.cs | 4 +-- .../Database/RedisDatabase.Scripts.cs | 4 +-- .../Database/RedisDatabase.Sets.cs | 4 +-- .../Database/RedisDatabase.Sort.cs | 4 +-- .../Database/RedisDatabase.SortedSets.cs | 4 +-- .../Database/RedisDatabase.Streams.cs | 4 +-- .../Database/RedisDatabase.Strings.cs | 4 +-- .../Database/RedisDatabase.VectorSets.cs | 4 +-- .../Groups/Arrays.Methods.cs | 4 +-- .../Groups/Bitmaps.Methods.cs | 4 +-- .../Groups/Config.Methods.cs | 4 +-- .../Groups/Diagnostics.Methods.cs | 10 ++++--- .../Groups/Geospatial.Methods.cs | 4 +-- .../Groups/Hashes.Methods.cs | 4 +-- .../Groups/HyperLogLog.Methods.cs | 11 ++++--- .../Groups/Keys.Methods.cs | 4 +-- src/StackExchange.Redis/Groups/Keys.Sort.cs | 4 +-- .../Groups/Lists.Methods.cs | 4 +-- .../Groups/PubSub.Methods.cs | 4 +-- .../Groups/Scripts.Methods.cs | 4 +-- .../Groups/Sets.Methods.cs | 4 +-- .../Groups/SortedSets.Methods.cs | 4 +-- .../Groups/Streams.Methods.cs | 19 +++++++----- .../Groups/Streams.Types.cs | 10 ++++--- .../Groups/Strings.Methods.cs | 4 +-- .../Groups/VectorSets.Methods.cs | 4 +-- src/StackExchange.Redis/Protocol/RespWire.cs | 2 +- src/StackExchange.Redis/ServerEndPoint.cs | 6 +++- 45 files changed, 152 insertions(+), 109 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 2556fa0ae..06388f21b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -50,7 +50,7 @@ - + diff --git a/src/StackExchange.Redis/Caching/CacheOptions.cs b/src/StackExchange.Redis/Caching/CacheOptions.cs index ce3b1df5f..f585a6800 100644 --- a/src/StackExchange.Redis/Caching/CacheOptions.cs +++ b/src/StackExchange.Redis/Caching/CacheOptions.cs @@ -103,7 +103,11 @@ public sealed class CacheOptions /// public int? MaxPayloadBytes { - get => _maxPayloadBytes; + get + { + return _maxPayloadBytes; + } + init => _maxPayloadBytes = value is null or > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The maximum payload size must be positive, or null for no limit."); @@ -137,7 +141,11 @@ public int? MaxPayloadBytes /// public long? MaxBytes { - get => _maxBytes; + get + { + return _maxBytes; + } + init => _maxBytes = value is null or > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The memory budget must be positive, or null for no limit."); @@ -155,7 +163,11 @@ public long? MaxBytes /// public int? MaxEntries { - get => _maxEntries; + get + { + return _maxEntries; + } + init => _maxEntries = value is null or > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The entry limit must be positive, or null for no limit."); @@ -190,7 +202,11 @@ public int? MaxEntries /// public int EvictionSampleSize { - get => _evictionSampleSize; + get + { + return _evictionSampleSize; + } + init => _evictionSampleSize = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value), "The eviction sample size must be positive."); @@ -268,7 +284,11 @@ public int EvictionSampleSize /// public IReadOnlyList Prefixes { - get => _prefixes; + get + { + return _prefixes; + } + init { _prefixes = value ?? throw new ArgumentNullException(nameof(value)); diff --git a/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs b/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs index fac3d396c..3b6caf9a2 100644 --- a/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs +++ b/src/StackExchange.Redis/Caching/RespClientCache.Slabs.cs @@ -6,9 +6,9 @@ namespace StackExchange.Redis.Caching { - /// + /// /// Where cached replies live: carved, in fill order, out of large arrays ("slabs") that the GC owns. - /// + /// /// /// /// Why not one array per reply. That made a hit free of shared writes - the GC is the reference count, so diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs b/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs index 8effd0288..6229aefc7 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.FeatureFlags.cs @@ -176,10 +176,10 @@ public static bool GetFeatureFlag(string flag) /// there is no separate subscription connection, so asking about answers /// about the shared one. /// - bool? IInternalConnectionMultiplexer.IsSyncReader(EndPoint endpoint, ConnectionType connectionType) - => ConnectionsIfCreated?.IsDedicatedThread(endpoint, connectionType, writer: false); + bool? IInternalConnectionMultiplexer.IsSyncReader(EndPoint endPoint, ConnectionType type) + => ConnectionsIfCreated?.IsDedicatedThread(endPoint, type, writer: false); /// As IsSyncReader, for the writer. - bool? IInternalConnectionMultiplexer.IsSyncWriter(EndPoint endpoint, ConnectionType connectionType) - => ConnectionsIfCreated?.IsDedicatedThread(endpoint, connectionType, writer: true); + bool? IInternalConnectionMultiplexer.IsSyncWriter(EndPoint endPoint, ConnectionType type) + => ConnectionsIfCreated?.IsDedicatedThread(endPoint, type, writer: true); } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index e6053b743..16c194ede 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -1441,7 +1441,8 @@ public IDatabase GetDatabase(int db = -1, object? asyncState = null) // if there's no async-state, and the DB is suitable, we can hand out a re-used instance return (asyncState == null && db <= MaxCachedDatabaseInstance) - ? GetCachedDatabaseInstance(db) : Surface(db, asyncState); + ? GetCachedDatabaseInstance(db) + : Surface(db, asyncState); } /// A database over the new core's connections. @@ -2902,12 +2903,12 @@ private void DisposeAndClearServers() } } - long? IInternalConnectionMultiplexer.GetConnectionId(EndPoint endpoint, ConnectionType type) + long? IInternalConnectionMultiplexer.GetConnectionId(EndPoint endPoint, ConnectionType type) // the core's connection, which is the one actually carrying the commands and the only one the // server can name. Asking must not dial: while both cores existed this asked a v3 bridge first, // and creating one here connected it - a whole v3 handshake on a third socket just to answer a // question - => ConnectionsIfCreated?.ConnectionId(endpoint, type); + => ConnectionsIfCreated?.ConnectionId(endPoint, type); internal uint UpdateLatency() { diff --git a/src/StackExchange.Redis/Contexts/RespConnectionManager.cs b/src/StackExchange.Redis/Contexts/RespConnectionManager.cs index 55418893d..df04e0100 100644 --- a/src/StackExchange.Redis/Contexts/RespConnectionManager.cs +++ b/src/StackExchange.Redis/Contexts/RespConnectionManager.cs @@ -137,9 +137,11 @@ internal RespConnectionManager(ConnectionMultiplexer multiplexer) /// /// private static RespClusterState SeedTopology(ConnectionMultiplexer multiplexer) - => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster ? RespClusterState.Yes - : multiplexer.IsConnected ? RespClusterState.No - : RespClusterState.Unknown; + => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster + ? RespClusterState.Yes + : multiplexer.IsConnected + ? RespClusterState.No + : RespClusterState.Unknown; internal bool RoutesBySlotForTest => _topology.RoutesBySlot; diff --git a/src/StackExchange.Redis/Contexts/RespHandshake.cs b/src/StackExchange.Redis/Contexts/RespHandshake.cs index 25ddaf083..f0b3e35c9 100644 --- a/src/StackExchange.Redis/Contexts/RespHandshake.cs +++ b/src/StackExchange.Redis/Contexts/RespHandshake.cs @@ -1564,9 +1564,11 @@ public HelloReply Parse(ref RespReader reader) else if (field == HelloField.Mode && reader.IsScalar) { // "standalone" | "sentinel" | "cluster"; only the last changes routing - mode = reader.Is("cluster"u8) ? ServerType.Cluster - : reader.Is("sentinel"u8) ? ServerType.Sentinel - : ServerType.Standalone; + mode = reader.Is("cluster"u8) + ? ServerType.Cluster + : reader.Is("sentinel"u8) + ? ServerType.Sentinel + : ServerType.Standalone; } else { diff --git a/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs b/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs index 9dc880783..c007c3290 100644 --- a/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs +++ b/src/StackExchange.Redis/Contexts/RespPayloadOperation.cs @@ -288,7 +288,11 @@ internal string CommandAndKey /// internal string? UnroutableRedirectMessage { - get => _redirectUnroutable ? _redirectMessage : null; + get + { + return _redirectUnroutable ? _redirectMessage : null; + } + set { _redirectMessage = value; @@ -319,7 +323,11 @@ internal string? UnroutableRedirectMessage /// internal string? DeclinedRedirectMessage { - get => _redirectUnroutable ? null : _redirectMessage; + get + { + return _redirectUnroutable ? null : _redirectMessage; + } + set { _redirectMessage = value; diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs b/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs index 61f8b1adf..b39c1ba7d 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Arrays.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The array commands, where they have moved to the RESP context surface. - /// + /// /// /// The whole family, ARGREP included: now renders its predicates /// through either writer, so the decision that was owed about that type is made. Arrays become spans diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs b/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs index 5d845bba3..260fc5f1d 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Bitmaps.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The bitmap commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The old surface calls these StringBitXxx and the new one groups them under diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs b/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs index e2e77055a..8200c0466 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Execute.cs @@ -5,9 +5,9 @@ namespace StackExchange.Redis { - /// + /// /// The ad-hoc escape hatch, where it has moved to the RESP context surface. - /// + /// /// /// /// Its own file rather than a command group, because it is not one: this is the route for commands diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs b/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs index 062072cd2..ffffa98ef 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Geospatial.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The geospatial commands, where they have moved to the RESP context surface. - /// + /// /// /// /// GeoRadius is the interesting adapter: the group has no method for it, because diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs b/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs index 3c48fb7ac..d2a30b315 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Hashes.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The hash commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The clearest illustration so far of what the new surface costs the old one: nothing. Six diff --git a/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs b/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs index a86e67e9a..52d039734 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.HyperLogLog.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The HyperLogLog commands, where they have moved to the RESP context surface. - /// + /// /// /// The old surface spells every one of these twice - a fixed-arity form and an array form - and the /// group collapses both onto the span, so the pairs meet again here. HyperLogLogMerge is the diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs b/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs index 330ec187c..9430e902b 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Keys.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The key commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The spelling of RespSurface.Keys.cs; see diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs b/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs index 8b51c4d8c..d445e5170 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Lists.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The list commands, where they have moved to the RESP context surface. - /// + /// /// /// /// ListRightPopLeftPush is the interesting adapter here: the group has no method of its own for diff --git a/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs b/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs index 6d1e02b1e..5f936f086 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.PubSub.cs @@ -2,9 +2,9 @@ namespace StackExchange.Redis { - /// + /// /// PUBLISH, where it has moved to the RESP context surface. - /// + /// /// /// /// The last two members SER352 was counting, and the ones that took the longest to be sure about. diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs b/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs index 4a572681a..eb6e7d880 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Scans.cs @@ -5,9 +5,9 @@ namespace StackExchange.Redis; -/// +/// /// The cursor scans, over the context surface. -/// +/// /// /// /// These are hand-written because [AutoDatabase] skips them, and it skips them for a real diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs b/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs index 0e0bbd5fc..b051603c2 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Scripts.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis; -/// +/// /// The scripting commands, over the context surface. -/// +/// /// /// /// These keep the shipped wire behaviour, which means they DO pair. This paragraph previously said diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs b/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs index eef4e88dd..23f42f546 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Sets.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The set commands, where they have moved to the RESP context surface. - /// + /// /// /// The group where the old surface's fixed-arity twins disappear entirely: SetCombine and /// SetCombineAndStore each have a (first, second) overload that existed only because diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs b/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs index 6c5d7e41e..003bc57db 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Sort.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// SORT, where it has moved to the RESP context surface. - /// + /// /// /// Four members and one renderer. The group is Keys rather than a group of its own, because /// SORT takes a list, a set or a sorted set and Redis files it under the generic commands. diff --git a/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs b/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs index 81c2ba0d8..f683c1436 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.SortedSets.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The sorted-set commands, where they have moved to the RESP context surface. - /// + /// /// /// /// The biggest adapter file, and mostly for one reason: SortedSetAdd alone has six overloads diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs b/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs index 11280dc58..722637c8f 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Streams.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The stream commands, where they have moved to the RESP context surface. - /// + /// /// /// The scalar half only; the reads return composites holding arrays and have not moved yet. Note the /// array parameters here: the old signatures promise RedisValue[], so this is where an array diff --git a/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs b/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs index c2b5f19c6..502af10c6 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.Strings.cs @@ -5,9 +5,9 @@ namespace StackExchange.Redis { - /// + /// /// The string commands, where they have moved to the RESP context surface. - /// + /// /// /// /// One file per command group, named to match the group it adapts - RespSurface.Strings.cs diff --git a/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs b/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs index 3dd0a758d..be3fd9880 100644 --- a/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs +++ b/src/StackExchange.Redis/Database/RedisDatabase.VectorSets.cs @@ -3,9 +3,9 @@ namespace StackExchange.Redis { - /// + /// /// The vector-set commands, where they have moved to the RESP context surface. - /// + /// /// /// The thinnest adapter of the lot, because the old surface was already lease-shaped: every member is /// a rename plus a writable-lease sibling, and there is no array anywhere except diff --git a/src/StackExchange.Redis/Groups/Arrays.Methods.cs b/src/StackExchange.Redis/Groups/Arrays.Methods.cs index 79971d680..56a5b8d2a 100644 --- a/src/StackExchange.Redis/Groups/Arrays.Methods.cs +++ b/src/StackExchange.Redis/Groups/Arrays.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The arrays commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs b/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs index f1dfe0708..432562b80 100644 --- a/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs +++ b/src/StackExchange.Redis/Groups/Bitmaps.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The bitmaps commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Config.Methods.cs b/src/StackExchange.Redis/Groups/Config.Methods.cs index 9f0cd01d9..03d43d978 100644 --- a/src/StackExchange.Redis/Groups/Config.Methods.cs +++ b/src/StackExchange.Redis/Groups/Config.Methods.cs @@ -6,9 +6,9 @@ namespace StackExchange.Redis; -/// +/// /// The CONFIG commands. -/// +/// /// public static partial class Config { diff --git a/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs b/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs index feef97ad7..cc496ed5f 100644 --- a/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs +++ b/src/StackExchange.Redis/Groups/Diagnostics.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The commands that ask a server about itself. -/// +/// /// public static partial class Diagnostics { @@ -616,7 +616,8 @@ LatencyHistoryEntry[] IRespHandler.Parse(ref RespReader r if (!reader.IsAggregate) throw new RespException("Unexpected LATENCY HISTORY reply."); return reader.ReadPastArray( static (ref RespReader r) => LatencyHistoryEntry.TryParseEntry(ref r, out var parsed) - ? parsed : throw new RespException("Unexpected LATENCY HISTORY element."), + ? parsed + : throw new RespException("Unexpected LATENCY HISTORY element."), scalar: false) ?? []; } @@ -625,7 +626,8 @@ LatencyLatestEntry[] IRespHandler.Parse(ref RespReader rea if (!reader.IsAggregate) throw new RespException("Unexpected LATENCY LATEST reply."); return reader.ReadPastArray( static (ref RespReader r) => LatencyLatestEntry.TryParseEntry(ref r, out var parsed) - ? parsed : throw new RespException("Unexpected LATENCY LATEST element."), + ? parsed + : throw new RespException("Unexpected LATENCY LATEST element."), scalar: false) ?? []; } } diff --git a/src/StackExchange.Redis/Groups/Geospatial.Methods.cs b/src/StackExchange.Redis/Groups/Geospatial.Methods.cs index a9bb3838e..8b57a559b 100644 --- a/src/StackExchange.Redis/Groups/Geospatial.Methods.cs +++ b/src/StackExchange.Redis/Groups/Geospatial.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The geospatial commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Hashes.Methods.cs b/src/StackExchange.Redis/Groups/Hashes.Methods.cs index 6a9255fec..41dfedc95 100644 --- a/src/StackExchange.Redis/Groups/Hashes.Methods.cs +++ b/src/StackExchange.Redis/Groups/Hashes.Methods.cs @@ -11,9 +11,9 @@ namespace StackExchange.Redis; -/// +/// /// The hashes commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs b/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs index 6f2258df3..67694cc08 100644 --- a/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs +++ b/src/StackExchange.Redis/Groups/HyperLogLog.Methods.cs @@ -8,9 +8,9 @@ namespace StackExchange.Redis; -/// +/// /// The hyperloglog commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. @@ -85,9 +85,8 @@ private static CommandFlags CountFlags(RespContext context, in RedisKey key, Com // only a KNOWN version demotes this. The old surface has the same rule for the same reason - // it demotes only when a server was actually selected - so a context that cannot ask keeps the // caller's routing, and the worst case is the one the caller asked for. - return context.TryGetFeatures(RedisCommand.PFCOUNT, in key, flags, out var features) - && !features.HyperLogLogCountReplicaSafe - ? CommandFlagsInternal.DemandPrimary(flags, RedisCommand.PFCOUNT) - : flags; + var replicaUnsafe = context.TryGetFeatures(RedisCommand.PFCOUNT, in key, flags, out var features) + && !features.HyperLogLogCountReplicaSafe; + return replicaUnsafe ? CommandFlagsInternal.DemandPrimary(flags, RedisCommand.PFCOUNT) : flags; } } diff --git a/src/StackExchange.Redis/Groups/Keys.Methods.cs b/src/StackExchange.Redis/Groups/Keys.Methods.cs index 1c02668ef..db3c812d0 100644 --- a/src/StackExchange.Redis/Groups/Keys.Methods.cs +++ b/src/StackExchange.Redis/Groups/Keys.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The keys commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Keys.Sort.cs b/src/StackExchange.Redis/Groups/Keys.Sort.cs index fffc60696..5828e2b70 100644 --- a/src/StackExchange.Redis/Groups/Keys.Sort.cs +++ b/src/StackExchange.Redis/Groups/Keys.Sort.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// SORT and SORT_RO, which hang off the key group rather than one of their own. -/// +/// /// /// A group file without a group type, and the reason the layout is per group rather than /// per file: SORT takes a key and belongs to , so it lives in that partial. It diff --git a/src/StackExchange.Redis/Groups/Lists.Methods.cs b/src/StackExchange.Redis/Groups/Lists.Methods.cs index 6de56575d..01bb67261 100644 --- a/src/StackExchange.Redis/Groups/Lists.Methods.cs +++ b/src/StackExchange.Redis/Groups/Lists.Methods.cs @@ -9,9 +9,9 @@ namespace StackExchange.Redis; -/// +/// /// The lists commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/PubSub.Methods.cs b/src/StackExchange.Redis/Groups/PubSub.Methods.cs index 59e1bc476..cbe51929e 100644 --- a/src/StackExchange.Redis/Groups/PubSub.Methods.cs +++ b/src/StackExchange.Redis/Groups/PubSub.Methods.cs @@ -7,9 +7,9 @@ namespace StackExchange.Redis; -/// +/// /// The publish/subscribe commands. -/// +/// /// public static partial class PubSub { diff --git a/src/StackExchange.Redis/Groups/Scripts.Methods.cs b/src/StackExchange.Redis/Groups/Scripts.Methods.cs index d8f88cbbd..3da80a420 100644 --- a/src/StackExchange.Redis/Groups/Scripts.Methods.cs +++ b/src/StackExchange.Redis/Groups/Scripts.Methods.cs @@ -11,9 +11,9 @@ namespace StackExchange.Redis; -/// +/// /// The scripts commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Sets.Methods.cs b/src/StackExchange.Redis/Groups/Sets.Methods.cs index 88dd09c4e..a10bf9c42 100644 --- a/src/StackExchange.Redis/Groups/Sets.Methods.cs +++ b/src/StackExchange.Redis/Groups/Sets.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The sets commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/SortedSets.Methods.cs b/src/StackExchange.Redis/Groups/SortedSets.Methods.cs index 4bf0bb061..6dd56d716 100644 --- a/src/StackExchange.Redis/Groups/SortedSets.Methods.cs +++ b/src/StackExchange.Redis/Groups/SortedSets.Methods.cs @@ -11,9 +11,9 @@ namespace StackExchange.Redis; -/// +/// /// The sortedsets commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/Streams.Methods.cs b/src/StackExchange.Redis/Groups/Streams.Methods.cs index 8b4abe8c4..c725d8dbc 100644 --- a/src/StackExchange.Redis/Groups/Streams.Methods.cs +++ b/src/StackExchange.Redis/Groups/Streams.Methods.cs @@ -8,9 +8,9 @@ namespace StackExchange.Redis; -/// +/// /// The stream commands: target.Streams.LengthAsync(...). -/// +/// /// /// /// Here rather than in a single surface-wide class, so that a group is one place: its type, its @@ -207,11 +207,13 @@ StreamPendingMessageInfo[] IRespHandler.Parse(ref Re StreamAutoClaimResult IRespHandler.Parse(ref RespReader reader) => RespParsers.TryParseStreamAutoClaim(ref reader, allowJaggedFields: true, out var value) - ? value : throw Unexpected("XAUTOCLAIM", ref reader); + ? value + : throw Unexpected("XAUTOCLAIM", ref reader); StreamAutoClaimIdsOnlyResult IRespHandler.Parse(ref RespReader reader) => RespParsers.TryParseStreamAutoClaimIdsOnly(ref reader, out var value) - ? value : throw Unexpected("XAUTOCLAIM", ref reader); + ? value + : throw Unexpected("XAUTOCLAIM", ref reader); RedisStream[] IRespHandler.Parse(ref RespReader reader) => RespParsers.ParseRedisStreams(ref reader, reader.Prefix == RespPrefix.Map, allowJaggedFields: true); @@ -799,10 +801,11 @@ internal static ValueTask ReadGroupArray( /// repeat. Raised here because only here are the positions visible; v3 did the same in its message. /// private static CommandFlags ReadGroupFlags(CommandFlags flags, RedisValue? position, TimeSpan? claimMinIdleTime) - => claimMinIdleTime is null && position is { } explicitPosition - && StreamPosition.Resolve(explicitPosition, RedisCommand.XREADGROUP) != StreamConstants.UndeliveredMessages - ? flags.WithRetryCategory(CommandFlags.CommandRetryReadOnly) - : flags; + { + var readOnly = claimMinIdleTime is null && position is { } explicitPosition + && StreamPosition.Resolve(explicitPosition, RedisCommand.XREADGROUP) != StreamConstants.UndeliveredMessages; + return readOnly ? flags.WithRetryCategory(CommandFlags.CommandRetryReadOnly) : flags; + } /// /// For several streams, a single > anywhere consumes, so all of them must be explicit. diff --git a/src/StackExchange.Redis/Groups/Streams.Types.cs b/src/StackExchange.Redis/Groups/Streams.Types.cs index 9ff0d4e50..484cb1f1a 100644 --- a/src/StackExchange.Redis/Groups/Streams.Types.cs +++ b/src/StackExchange.Redis/Groups/Streams.Types.cs @@ -6,9 +6,9 @@ namespace StackExchange.Redis; -/// +/// /// The stream reply shapes, as windows over the reply buffer. -/// +/// /// /// /// Nested in the group, because these types are the group's. A stream entry means nothing outside @@ -355,7 +355,8 @@ public StreamAutoClaimResult ToStreamAutoClaimResult() var reader = GetReader(); reader.MoveNext(); return RespParsers.TryParseStreamAutoClaim(ref reader, allowJaggedFields: true, out var value) - ? value : StreamAutoClaimResult.Null; + ? value + : StreamAutoClaimResult.Null; } /// @@ -427,7 +428,8 @@ public StreamAutoClaimIdsOnlyResult ToStreamAutoClaimIdsOnlyResult() var reader = GetReader(); reader.MoveNext(); return RespParsers.TryParseStreamAutoClaimIdsOnly(ref reader, out var value) - ? value : StreamAutoClaimIdsOnlyResult.Null; + ? value + : StreamAutoClaimIdsOnlyResult.Null; } /// diff --git a/src/StackExchange.Redis/Groups/Strings.Methods.cs b/src/StackExchange.Redis/Groups/Strings.Methods.cs index 5d18255ec..a20fd062d 100644 --- a/src/StackExchange.Redis/Groups/Strings.Methods.cs +++ b/src/StackExchange.Redis/Groups/Strings.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The strings commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Groups/VectorSets.Methods.cs b/src/StackExchange.Redis/Groups/VectorSets.Methods.cs index 8c7e367c5..502894ed9 100644 --- a/src/StackExchange.Redis/Groups/VectorSets.Methods.cs +++ b/src/StackExchange.Redis/Groups/VectorSets.Methods.cs @@ -10,9 +10,9 @@ namespace StackExchange.Redis; -/// +/// /// The vectorsets commands. -/// +/// /// /// Here rather than in a single surface-wide class, so that a group is one place. The extension methods /// bind by namespace, and the namespace is StackExchange.Redis, so this costs a caller nothing. diff --git a/src/StackExchange.Redis/Protocol/RespWire.cs b/src/StackExchange.Redis/Protocol/RespWire.cs index ed21aeadd..5053b38d4 100644 --- a/src/StackExchange.Redis/Protocol/RespWire.cs +++ b/src/StackExchange.Redis/Protocol/RespWire.cs @@ -153,7 +153,7 @@ internal static void WriteMultiBulkHeader(IBufferWriter writer, long count internal static void WriteMultiBulkHeader(IBufferWriter writer, long count, RespPrefix prefix) { - if ((prefix is RespPrefix.Map or RespPrefix.Attribute) & count > 0) + if ((prefix is RespPrefix.Map or RespPrefix.Attribute) & (count > 0)) { if ((count & 1) != 0) Throw(prefix, count); count >>= 1; diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index 9d5a59459..4dc563da5 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -219,7 +219,11 @@ public ServerType ServerType public bool IsReplica { - get => isReplica; + get + { + return isReplica; + } + set { var changed = isReplica != value; From 25f7b1460d7627f1f341e5f2ef224abc22e59064 Mon Sep 17 00:00:00 2001 From: Marc Gravell Date: Fri, 23 Oct 2026 17:47:37 +0100 Subject: [PATCH 22/22] Merge fix-up: restore v4's CI.yml (the --theirs resolution had replaced the whole file), and the report-github reporter for the two small test projects - CI.yml: the merge took main's file wholesale for a one-hunk conflict, dropping v4's Windows server setup - the diskless replica load and the wait for every replica to sync and be listed by CLUSTER SLOTS - as well as the v4 branch triggers and the 30-minute test step timeout. Without the replica wait, no replica on the Windows job ever syncs, CLUSTER SLOTS lists none, and the replica-routing tests fail ("only 0 slots carried replicas"). This is v4's file with main's one change applied: the Microsoft.Testing.Platform `dotnet test` invocation. VSTest's --blame-hang has no platform equivalent without the HangDump extension; the step timeout stays as the backstop. - RESPite.Tests and Build.Tests: the CI action passes --report-github to every test project, which is the GitHubActionsTestLogger package's platform extension; the merge had dropped that reference from these two, so their steps exited 5 (invalid arguments, zero tests ran). Verified locally with the action's exact flags: 1960 and 188 pass. --- .github/workflows/CI.yml | 76 ++++++++++++++++--- tests/RESPite.Tests/RESPite.Tests.csproj | 1 + .../StackExchange.Redis.Build.Tests.csproj | 1 + 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1522cf45f..7d8a9dfe1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,7 +4,7 @@ run-name: "CI${{ github.event.inputs.client-libs-test-image-tag != '' && format( on: pull_request: push: - branches: [ 'main', 'v3' ] + branches: [ 'main', 'v3', 'v4' ] paths: - '**' - '!/docs/*' # Don't run workflow when files are only in the /docs directory @@ -145,23 +145,33 @@ jobs: TOPOLOGY='${{ github.event.inputs.windows-topology || 'full' }}' echo "topology: $TOPOLOGY" - redis-server ../Basic/primary-6379.conf & - redis-server ../Basic/replica-6380.conf & - redis-server ../Basic/secure-6381.conf & - redis-server ../Failover/primary-6382.conf & - redis-server ../Failover/replica-6383.conf & + # Replicas load the initial sync from the socket, never from disk. These data directories + # sit on the Windows mount (/mnt/...), where a replica receives the RDB to a file and then + # fails to load it ("Failed trying to load the MASTER synchronization DB from disk"), retrying + # forever - so no replica on this job had ever synced. That went unnoticed while the client + # read replicas from CLUSTER NODES, which lists them regardless; CLUSTER SLOTS rightly leaves + # out a replica that has never synced, so the new core saw a cluster with no replicas. + LOAD="--repl-diskless-load swapdb" + + redis-server ../Basic/primary-6379.conf $LOAD & + redis-server ../Basic/replica-6380.conf $LOAD & + redis-server ../Basic/secure-6381.conf $LOAD & + redis-server ../Failover/primary-6382.conf $LOAD & + redis-server ../Failover/replica-6383.conf $LOAD & PORTS="6379 6380 6381 6382 6383" + REPLICAS="6380 6383" if [ "$TOPOLOGY" = "full" ]; then for p in 7000 7001 7002 7003 7004 7005; do - redis-server "../Cluster/cluster-$p.conf" --dir ../Cluster & + redis-server "../Cluster/cluster-$p.conf" --dir ../Cluster $LOAD & done - redis-server ../Sentinel/redis-7010.conf & - redis-server ../Sentinel/redis-7011.conf & + redis-server ../Sentinel/redis-7010.conf $LOAD & + redis-server ../Sentinel/redis-7011.conf $LOAD & for p in 26379 26380 26381; do redis-server "../Sentinel/sentinel-$p.conf" --sentinel & done PORTS="$PORTS 7000 7001 7002 7003 7004 7005 7010 7011 26379 26380 26381" + REPLICAS="$REPLICAS 7003 7004 7005 7011" fi # Poll rather than `sleep 5`: on a fast machine this returns almost immediately, and on a @@ -197,11 +207,53 @@ jobs: exit 1 fi + # ...and a replica that is listening but has never synced is just as misleading, and quieter: + # replica-routing tests then pass or fail on what the client happens to read, not on the + # client. Fatal for the same reason as above. + unsynced="" + for p in $REPLICAS; do + synced="" + for _ in $(seq 1 30); do + if redis-cli -p "$p" INFO replication 2>/dev/null | grep -q "master_link_status:up"; then + synced="yes"; break + fi + sleep 1 + done + if [ -n "$synced" ]; then + echo "replica $p synced" + else + echo "replica $p DID NOT SYNC:" + redis-cli -p "$p" INFO replication 2>&1 || true + unsynced="$unsynced $p" + fi + done + if [ -n "$unsynced" ]; then + echo "::error::replicas not synced:$unsynced" + exit 1 + fi + + # CLUSTER SLOTS lists a replica only once its primary has heard, by gossip, that it has an + # offset - a second or so after the sync above. Wait for all six nodes to be listed. + if [ "$TOPOLOGY" = "full" ]; then + listed=0 + for _ in $(seq 1 30); do + listed=$(redis-cli -p 7000 CLUSTER SLOTS 2>/dev/null | grep -c '^700[0-5]$' || true) + [ "$listed" -ge 6 ] && break + sleep 1 + done + echo "CLUSTER SLOTS lists $listed of 6 nodes" + if [ "$listed" -lt 6 ]; then + redis-cli -p 7000 CLUSTER NODES 2>&1 || true + echo "::error::CLUSTER SLOTS does not list every replica" + exit 1 + fi + fi + - name: .NET Build run: dotnet build Build.csproj -c Release /p:CI=true # The analyzer and the props that configures it reach consumers only through the package, and a # packaging regression is silent: no diagnostics, ever, for anybody, with a build that still succeeds. - # The real pack below runs only on pushes to main/v3, so check it here where PRs will see it too. + # The real pack below runs only on pushes to main/v3/v4, so check it here where PRs will see it too. - name: Verify package contents run: | $out = "${env:GITHUB_WORKSPACE}\.packcheck" @@ -222,6 +274,10 @@ jobs: exit 1 } - name: StackExchange.Redis.Tests + # A hung test otherwise holds the job until GitHub's six-hour limit; the step timeout is the backstop. + # (VSTest's --blame-hang named the hung test and killed it; the Microsoft.Testing.Platform runner has + # no equivalent without its HangDump extension, so for now the timeout is all there is.) + timeout-minutes: 30 run: | $exitCode = 0 foreach ($tfm in @("net10.0", "net481")) { diff --git a/tests/RESPite.Tests/RESPite.Tests.csproj b/tests/RESPite.Tests/RESPite.Tests.csproj index 74af96a40..b3cd2b4b2 100644 --- a/tests/RESPite.Tests/RESPite.Tests.csproj +++ b/tests/RESPite.Tests/RESPite.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj index 060064481..6bb6e92a7 100644 --- a/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj +++ b/tests/StackExchange.Redis.Build.Tests/StackExchange.Redis.Build.Tests.csproj @@ -30,6 +30,7 @@ +