Skip to content

Commit 61b8d6c

Browse files
Yury-FridlyandYury-Fridlyand
committed
Add e2e test
Signed-off-by: Yury-Fridlyand <[email protected]>
1 parent bd94403 commit 61b8d6c

File tree

3 files changed

+242
-1
lines changed

3 files changed

+242
-1
lines changed

‎sources/Valkey.Glide/Internals/FFI.methods.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
// Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0
22

33
// check https://stackoverflow.com/a/77455034 if you're getting analyzer error (using is unnecessary)
4+
#if NET8_0_OR_GREATER
45
using System.Runtime.CompilerServices;
6+
#endif
57
using System.Runtime.InteropServices;
68

79
namespace Valkey.Glide.Internals;

‎tests/Valkey.Glide.IntegrationTests/AzAffinityTests.cs

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
// Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0
2+
3+
using static Valkey.Glide.Commands.Options.InfoOptions;
4+
using static Valkey.Glide.ConnectionConfiguration;
5+
using static Valkey.Glide.Route;
6+
7+
namespace Valkey.Glide.IntegrationTests;
8+
9+
public class AzAffinityTests(TestConfiguration config)
10+
{
11+
public TestConfiguration Config { get; } = config;
12+
13+
private static GlideClusterClient CreateAzTestClient(string az)
14+
{
15+
ClusterClientConfiguration config = TestConfiguration.DefaultClusterClientConfig()
16+
.WithReadFrom(new(ReadFromStrategy.AzAffinity, az))
17+
.WithRequestTimeout(TimeSpan.FromSeconds(2))
18+
.Build();
19+
return GlideClusterClient.CreateClient(config).GetAwaiter().GetResult();
20+
}
21+
22+
private static GlideClusterClient CreateAzAffinityReplicasAndPrimaryTestClient(string az)
23+
{
24+
ClusterClientConfiguration config = TestConfiguration.DefaultClusterClientConfig()
25+
.WithReadFrom(new(ReadFromStrategy.AzAffinityReplicasAndPrimary, az))
26+
.WithRequestTimeout(TimeSpan.FromSeconds(2))
27+
.Build();
28+
return GlideClusterClient.CreateClient(config).GetAwaiter().GetResult();
29+
}
30+
31+
[Theory(DisableDiscoveryEnumeration = true)]
32+
[MemberData(nameof(Config.TestClusterClients), MemberType = typeof(TestConfiguration))]
33+
public async Task TestRoutingWithAzAffinityStrategyTo1Replica(GlideClusterClient configClient)
34+
{
35+
Assert.SkipWhen(TestConfiguration.SERVER_VERSION < new Version("8.0.0"), "AZ affinity requires server version 8.0.0 or higher");
36+
37+
const string az = "us-east-1a";
38+
const int getCalls = 3;
39+
string getCmdStat = $"cmdstat_get:calls={getCalls}";
40+
41+
// Reset the availability zone for all nodes
42+
await configClient.CustomCommand(["config", "set", "availability-zone", ""], AllNodes);
43+
await configClient.CustomCommand(["config", "resetstat"], AllNodes);
44+
45+
// 12182 is the slot of "foo"
46+
await configClient.CustomCommand(["config", "set", "availability-zone", az], new SlotIdRoute(12182, SlotType.Replica));
47+
48+
using GlideClusterClient azTestClient = CreateAzTestClient(az);
49+
50+
for (int i = 0; i < getCalls; i++)
51+
{
52+
await azTestClient.StringGetAsync("foo");
53+
}
54+
55+
ClusterValue<string> infoResult = await azTestClient.Info([Section.SERVER, Section.COMMANDSTATS], AllNodes);
56+
57+
// Check that only the replica with az has all the GET calls
58+
int matchingEntriesCount = 0;
59+
foreach (string value in infoResult.MultiValue.Values)
60+
{
61+
if (value.Contains(getCmdStat) && value.Contains(az))
62+
{
63+
matchingEntriesCount++;
64+
}
65+
}
66+
Assert.Equal(1, matchingEntriesCount);
67+
68+
// Check that the other replicas have no availability zone set
69+
int changedAzCount = 0;
70+
foreach (string value in infoResult.MultiValue.Values)
71+
{
72+
if (value.Contains($"availability_zone:{az}"))
73+
{
74+
changedAzCount++;
75+
}
76+
}
77+
Assert.Equal(1, changedAzCount);
78+
}
79+
80+
[Theory(DisableDiscoveryEnumeration = true)]
81+
[MemberData(nameof(Config.TestClusterClients), MemberType = typeof(TestConfiguration))]
82+
public async Task TestRoutingBySlotToReplicaWithAzAffinityStrategyToAllReplicas(GlideClusterClient configClient)
83+
{
84+
Assert.SkipWhen(TestConfiguration.SERVER_VERSION < new Version("8.0.0"), "AZ affinity requires server version 8.0.0 or higher");
85+
86+
const string az = "us-east-1a";
87+
88+
// Reset the availability zone for all nodes
89+
await configClient.CustomCommand(["config", "set", "availability-zone", ""], AllNodes);
90+
await configClient.CustomCommand(["config", "resetstat"], AllNodes);
91+
92+
// Get Replica Count for current cluster
93+
ClusterValue<string> clusterInfo = await configClient.Info([Section.REPLICATION], new SlotKeyRoute("key", SlotType.Primary));
94+
int nReplicas = 0;
95+
foreach (string line in clusterInfo.SingleValue!.Split('\n'))
96+
{
97+
string[] parts = line.Split(':', 2);
98+
if (parts.Length == 2 && parts[0].Trim() == "connected_slaves")
99+
{
100+
nReplicas = int.Parse(parts[1].Trim());
101+
break;
102+
}
103+
}
104+
105+
int nGetCalls = 3 * nReplicas;
106+
string getCmdStat = "cmdstat_get:calls=3";
107+
108+
// Setting AZ for all Nodes
109+
await configClient.CustomCommand(["config", "set", "availability-zone", az], AllNodes);
110+
111+
using GlideClusterClient azTestClient = CreateAzTestClient(az);
112+
113+
ClusterValue<object?> azGetResult = await azTestClient.CustomCommand(["config", "get", "availability-zone"], AllNodes);
114+
foreach (object? value in azGetResult.MultiValue.Values)
115+
{
116+
object[]? configArray = value as object[];
117+
if (configArray != null && configArray.Length >= 2)
118+
{
119+
Assert.Equal(az, configArray[1]?.ToString());
120+
}
121+
}
122+
123+
// Execute GET commands
124+
for (int i = 0; i < nGetCalls; i++)
125+
{
126+
await azTestClient.StringGetAsync("foo");
127+
}
128+
129+
ClusterValue<string> infoResult = await azTestClient.Info([Section.ALL], AllNodes);
130+
131+
// Check that all replicas have the same number of GET calls
132+
int matchingEntriesCount = 0;
133+
foreach (string value in infoResult.MultiValue.Values)
134+
{
135+
if (value.Contains(getCmdStat) && value.Contains(az))
136+
{
137+
matchingEntriesCount++;
138+
}
139+
}
140+
Assert.Equal(nReplicas, matchingEntriesCount);
141+
}
142+
143+
[Fact]
144+
public async Task TestAzAffinityNonExistingAz()
145+
{
146+
Assert.SkipWhen(TestConfiguration.SERVER_VERSION < new Version("8.0.0"), "AZ affinity requires server version 8.0.0 or higher");
147+
148+
const int nGetCalls = 3;
149+
const int nReplicaCalls = 1;
150+
string getCmdStat = $"cmdstat_get:calls={nReplicaCalls}";
151+
152+
using GlideClusterClient azTestClient = CreateAzTestClient("non-existing-az");
153+
154+
// Reset stats
155+
await azTestClient.CustomCommand(["config", "resetstat"], AllNodes);
156+
157+
// Execute GET commands
158+
for (int i = 0; i < nGetCalls; i++)
159+
{
160+
await azTestClient.StringGetAsync("foo");
161+
}
162+
163+
ClusterValue<string> infoResult = await azTestClient.Info([Section.COMMANDSTATS], AllNodes);
164+
165+
// We expect the calls to be distributed evenly among the replicas
166+
int matchingEntriesCount = 0;
167+
foreach (string value in infoResult.MultiValue.Values)
168+
{
169+
if (value.Contains(getCmdStat))
170+
{
171+
matchingEntriesCount++;
172+
}
173+
}
174+
Assert.Equal(3, matchingEntriesCount);
175+
}
176+
177+
[Theory(DisableDiscoveryEnumeration = true)]
178+
[MemberData(nameof(Config.TestClusterClients), MemberType = typeof(TestConfiguration))]
179+
public async Task TestAzAffinityReplicasAndPrimaryRoutesToPrimary(GlideClusterClient configClient)
180+
{
181+
Assert.SkipWhen(TestConfiguration.SERVER_VERSION < new Version("8.0.0"), "AZ affinity requires server version 8.0.0 or higher");
182+
183+
const string az = "us-east-1a";
184+
const string otherAz = "us-east-1b";
185+
const int nGetCalls = 4;
186+
string getCmdStat = $"cmdstat_get:calls={nGetCalls}";
187+
188+
// Reset stats and set all nodes to otherAz
189+
await configClient.CustomCommand(["config", "resetstat"], AllNodes);
190+
await configClient.CustomCommand(["config", "set", "availability-zone", otherAz], AllNodes);
191+
192+
// Set primary for slot 12182 to az
193+
await configClient.CustomCommand(["config", "set", "availability-zone", az], new SlotIdRoute(12182, SlotType.Primary));
194+
195+
// Verify primary AZ
196+
ClusterValue<object?> primaryAzResult = await configClient.CustomCommand(["config", "get", "availability-zone"], new SlotIdRoute(12182, SlotType.Primary));
197+
object[]? primaryConfigArray = primaryAzResult.SingleValue as object[];
198+
if (primaryConfigArray != null && primaryConfigArray.Length >= 2)
199+
{
200+
Assert.Equal(az, primaryConfigArray[1]?.ToString());
201+
}
202+
203+
using GlideClusterClient azTestClient = CreateAzAffinityReplicasAndPrimaryTestClient(az);
204+
205+
// Execute GET commands
206+
for (int i = 0; i < nGetCalls; i++)
207+
{
208+
await azTestClient.StringGetAsync("foo");
209+
}
210+
211+
ClusterValue<string> infoResult = await azTestClient.Info([Section.ALL], AllNodes);
212+
213+
// Check that only the primary in the specified AZ handled all GET calls
214+
int matchingEntriesCount = 0;
215+
foreach (string value in infoResult.MultiValue.Values)
216+
{
217+
if (value.Contains(getCmdStat) && value.Contains(az) && value.Contains("role:master"))
218+
{
219+
matchingEntriesCount++;
220+
}
221+
}
222+
Assert.Equal(1, matchingEntriesCount);
223+
224+
// Verify total GET calls
225+
int totalGetCalls = 0;
226+
foreach (string value in infoResult.MultiValue.Values)
227+
{
228+
if (value.Contains("cmdstat_get:calls="))
229+
{
230+
int startIndex = value.IndexOf("cmdstat_get:calls=") + "cmdstat_get:calls=".Length;
231+
int endIndex = value.IndexOf(',', startIndex);
232+
if (endIndex == -1) endIndex = value.Length;
233+
int calls = int.Parse(value.Substring(startIndex, endIndex - startIndex));
234+
totalGetCalls += calls;
235+
}
236+
}
237+
Assert.Equal(nGetCalls, totalGetCalls);
238+
}
239+
}

‎tests/Valkey.Glide.IntegrationTests/TestConfiguration.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ private static void TestConsoleWriteLine(string message) =>
302302

303303
internal List<(string host, ushort port)> StartServer(bool cluster, bool tls = false, string? name = null)
304304
{
305-
string cmd = $"start {(cluster ? "--cluster-mode" : "")} {(tls ? " --tls" : "")} {(name != null ? " --prefix " + name : "")}";
305+
string cmd = $"start {(cluster ? "--cluster-mode" : "")} {(tls ? " --tls" : "")} {(name != null ? " --prefix " + name : "")} -r 3";
306306
return ParseHostsFromOutput(RunClusterManager(cmd, false));
307307
}
308308

0 commit comments

Comments
 (0)