Skip to content

Dynamic enablement of Per-Partition Automatic Failover #46477

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 31 commits into
base: main
Choose a base branch
from
Open

Dynamic enablement of Per-Partition Automatic Failover #46477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
907b003
Enabling PPAF dynamically.
jeet1995 Aug 21, 2025
88d24c0
Ensure atomicity when opt-ins for PPAF, PPCB and hedging are being mo…
jeet1995 Aug 21, 2025
740cded
Ensure atomicity when opt-ins for PPAF, PPCB and hedging are being mo…
jeet1995 Aug 21, 2025
ad1353b
Clear state (PPAF + PPCB) when PPAF config modifier callback is invoked.
jeet1995 Aug 21, 2025
a033bc8
Updated CHANGELOG.md
jeet1995 Sep 3, 2025
8ffdd5f
Updated CHANGELOG.md
jeet1995 Sep 3, 2025
8126742
Adding gateway response from non-responsive region in Gateway mode.
jeet1995 Sep 4, 2025
7db1c50
Fixing PerPartitionAutomaticFailoverE2ETests.
jeet1995 Sep 7, 2025
443de1f
Fixing PerPartitionAutomaticFailoverE2ETests.
jeet1995 Sep 8, 2025
7b499e2
Merge branch 'main' of github.com:jeet1995/azure-sdk-for-java into Pp…
jeet1995 Sep 8, 2025
b00a217
Fixing PerPartitionAutomaticFailoverE2ETests.
jeet1995 Sep 8, 2025
7aec10e
Merge branch 'main' of github.com:jeet1995/azure-sdk-for-java into Pp…
jeet1995 Sep 8, 2025
ee45562
Updated CHANGELOG.md
jeet1995 Sep 8, 2025
6c6c4b7
Merge branch 'main' of github.com:jeet1995/azure-sdk-for-java into Pp…
jeet1995 Sep 9, 2025
b00bfc4
Merge branch 'main' of github.com:jeet1995/azure-sdk-for-java into Pp…
jeet1995 Sep 11, 2025
39da8a2
Attempt to fix diagnostics.
jeet1995 Sep 11, 2025
241feb8
Have PPCB rely on user-enforced sys prop when PPAF is disabled on acc…
jeet1995 Sep 14, 2025
c49e853
Move return early logic before lock acquisition.
jeet1995 Sep 14, 2025
88f6fb7
Modify PPAF config change detection logic.
jeet1995 Sep 14, 2025
ad65018
Fixing a cross-region availability strategy test where if suppressReq…
jeet1995 Sep 14, 2025
975ccff
Fixing user agent suffix.
jeet1995 Sep 14, 2025
8af012e
Adjusting tests to accommodate 408-20008 in cases where the main requ…
jeet1995 Sep 18, 2025
79dc9f2
Adjust tests to validate user agents suffixed differently due to HTTP…
jeet1995 Sep 18, 2025
0c1a60d
Merge branch 'main' of github.com:jeet1995/azure-sdk-for-java into Pp…
jeet1995 Sep 18, 2025
03ab8d7
Fixing live tests.
jeet1995 Sep 22, 2025
6d646a2
Fixing `isPpafEnabled` flag in `CosmosDiagnostics`.
jeet1995 Sep 22, 2025
18300ab
Updated CHANGELOG.md
jeet1995 Sep 22, 2025
b066cf1
Attempt to fix memory leak.
jeet1995 Sep 22, 2025
29cbe77
Attempt to fix memory leak.
jeet1995 Sep 23, 2025
f10e3d1
Addressing review comments.
jeet1995 Oct 1, 2025
d7090ac
Merge branch 'main' of github.com:jeet1995/azure-sdk-for-java into Pp…
jeet1995 Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ public void beforeClass() {

UserAgentContainer userAgentContainer = new UserAgentContainer();
userAgentContainer.setSuffix(USER_AGENT_SUFFIX_GATEWAY_CLIENT);
this.gatewayClientUserAgent = userAgentContainer.getUserAgent();
this.gatewayClientUserAgent = generateHttp2OptedInUserAgentIfRequired(userAgentContainer.getUserAgent());

directClient = new CosmosClientBuilder()
.endpoint(TestConfigurations.HOST)
Expand All @@ -164,7 +164,7 @@ public void beforeClass() {
.directMode()
.buildClient();
userAgentContainer.setSuffix(USER_AGENT_SUFFIX_DIRECT_CLIENT);
this.directClientUserAgent = userAgentContainer.getUserAgent();
this.directClientUserAgent = generateHttp2OptedInUserAgentIfRequired(userAgentContainer.getUserAgent());

cosmosAsyncContainer = getSharedMultiPartitionCosmosContainer(this.gatewayClient.asyncClient());
cosmosAsyncDatabase = directClient.asyncClient().getDatabase(cosmosAsyncContainer.getDatabase().getId());
Expand Down Expand Up @@ -271,7 +271,6 @@ public void queryChangeFeedAllVersionsAndDeletes() {
FeedResponse<JsonNode> response = results.next();
String diagnostics = response.getCosmosDiagnostics().toString();
assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\"");
assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\"");
assertThat(diagnostics).contains("gatewayStatisticsList");
assertThat(diagnostics).contains("\"operationType\":\"ReadFeed\"");
assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\"");
Expand All @@ -298,7 +297,7 @@ public void queryChangeFeedIncrementalDirectMode() throws Exception {
FeedResponse<JsonNode> response = results.next();
String diagnostics = response.getCosmosDiagnostics().toString();
assertThat(diagnostics).contains("\"connectionMode\":\"DIRECT\"");
assertThat(diagnostics).contains("\"userAgent\":\"" + this.directClientUserAgent + "\"");
assertThat(diagnostics).contains("\"userAgent\":\"" + generateHttp2OptedInUserAgentIfRequired(this.directClientUserAgent) + "\"");
assertThat(diagnostics).contains("\"requestOperationType\":\"ReadFeed\"");
}
}
Expand All @@ -324,7 +323,6 @@ public void queryChangeFeedIncrementalGatewayMode() throws Exception {
assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\"");
assertThat(diagnostics).contains("gatewayStatisticsList");
assertThat(diagnostics).contains("\"operationType\":\"ReadFeed\"");
assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\"");
}
}

Expand All @@ -349,7 +347,6 @@ public void gatewayDiagnostics() throws Exception {
String diagnostics = createResponse.getDiagnostics().toString();
logger.info("DIAGNOSTICS: {}", diagnostics);
assertThat(diagnostics).contains("\"connectionMode\":\"GATEWAY\"");
assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\"");
assertThat(diagnostics).contains("gatewayStatisticsList");
assertThat(diagnostics).contains("\"operationType\":\"Create\"");
assertThat(diagnostics).contains("\"metaDataName\":\"CONTAINER_LOOK_UP\"");
Expand Down Expand Up @@ -392,7 +389,6 @@ public void gatewayDiagnosticsOnException() throws Exception {
assertThat(diagnostics).contains("gatewayStatisticsList");
assertThat(diagnostics).contains("\"statusCode\":404");
assertThat(diagnostics).contains("\"operationType\":\"Read\"");
assertThat(diagnostics).contains("\"userAgent\":\"" + this.gatewayClientUserAgent + "\"");
assertThat(diagnostics).contains("\"exceptionMessage\":\"Entity with the specified id does not exist in the system.");
assertThat(diagnostics).contains("\"exceptionResponseHeaders\"");
assertThat(diagnostics).doesNotContain("\"exceptionResponseHeaders\": \"{}\"");
Expand Down Expand Up @@ -1971,6 +1967,14 @@ private void validateChannelAcquisitionContext(CosmosDiagnostics diagnostics, bo
}
}

private String generateHttp2OptedInUserAgentIfRequired(String userAgent) {
if (Configs.isHttp2Enabled()) {
userAgent = userAgent + "|F10";
}

return userAgent;
}

private CosmosDiagnostics performDocumentOperation(
CosmosAsyncContainer cosmosAsyncContainer,
OperationType operationType,
Expand Down
66 changes: 41 additions & 25 deletions ...tests/src/test/java/com/azure/cosmos/FaultInjectionWithAvailabilityStrategyTestsBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2359,6 +2359,15 @@ public Object[][] testConfigs_queryAfterCreation() {
}
};

BiConsumer<CosmosDiagnosticsContext, Integer> validateCtxUtmostRegions =
(ctx, expectedNumberOfRegionsContacted) -> {
assertThat(ctx).isNotNull();
if (ctx != null) {
assertThat(ctx.getContactedRegionNames().size()).isGreaterThanOrEqualTo(1);
assertThat(ctx.getContactedRegionNames().size()).isLessThanOrEqualTo(expectedNumberOfRegionsContacted);
}
};

Consumer<CosmosDiagnosticsContext> validateCtxQueryPlan =
(ctx) -> {
assertThat(ctx).isNotNull();
Expand Down Expand Up @@ -2429,6 +2438,9 @@ public Object[][] testConfigs_queryAfterCreation() {
Consumer<CosmosDiagnosticsContext> validateCtxTwoRegions =
(ctx) -> validateCtxRegions.accept(ctx, TWO_REGIONS);

Consumer<CosmosDiagnosticsContext> validateCtxUtmostTwoRegions =
(ctx) -> validateCtxUtmostRegions.accept(ctx, TWO_REGIONS);

Consumer<CosmosDiagnosticsContext> validateCtxFirstRegionFailureSecondRegionSuccessfulSingleFeedResponse = (ctx) -> {
CosmosDiagnostics[] diagnostics = ctx.getDiagnostics().toArray(new CosmosDiagnostics[0]);
assertThat(diagnostics.length).isEqualTo(3);
Expand Down Expand Up @@ -3270,7 +3282,7 @@ public Object[][] testConfigs_queryAfterCreation() {
CosmosDiagnostics[] diagnostics = ctx.getDiagnostics().toArray(new CosmosDiagnostics[0]);

// Diagnostics of query attempt in first region not even available yet
assertThat(diagnostics.length).isEqualTo(2);
assertThat(diagnostics.length).isGreaterThanOrEqualTo(2);

// query plan on first region
assertThat(diagnostics[0].getContactedRegionNames().size()).isEqualTo(1);
Expand All @@ -3279,13 +3291,13 @@ public Object[][] testConfigs_queryAfterCreation() {
(ctx) -> {
assertThat(ctx.getDiagnostics()).isNotNull();
CosmosDiagnostics[] diagnostics = ctx.getDiagnostics().toArray(new CosmosDiagnostics[0]);
assertThat(diagnostics[1].getContactedRegionNames().size()).isEqualTo(1);
assertThat(diagnostics[1].getContactedRegionNames().iterator().next()).isEqualTo(SECOND_REGION_NAME);
assertThat(diagnostics[1].getFeedResponseDiagnostics()).isNotNull();
assertThat(diagnostics[1].getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(diagnostics[1].getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getContactedRegionNames().size()).isEqualTo(1);
assertThat(diagnostics[diagnostics.length - 1].getContactedRegionNames().iterator().next()).isEqualTo(SECOND_REGION_NAME);
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
ClientSideRequestStatistics[] clientStats =
diagnostics[1]
diagnostics[diagnostics.length - 1]
.getFeedResponseDiagnostics()
.getClientSideRequestStatistics()
.toArray(new ClientSideRequestStatistics[0]);
Expand All @@ -3301,17 +3313,17 @@ public Object[][] testConfigs_queryAfterCreation() {
}
),
ArrayUtils.toArray(
validateCtxSingleRegion,
validateCtxUtmostTwoRegions,
(ctx) -> {
assertThat(ctx.getDiagnostics()).isNotNull();
CosmosDiagnostics[] diagnostics = ctx.getDiagnostics().toArray(new CosmosDiagnostics[0]);
assertThat(diagnostics[0].getContactedRegionNames().size()).isEqualTo(1);
assertThat(diagnostics[0].getContactedRegionNames().iterator().next()).isEqualTo(SECOND_REGION_NAME);
assertThat(diagnostics[0].getFeedResponseDiagnostics()).isNotNull();
assertThat(diagnostics[0].getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(diagnostics[0].getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getContactedRegionNames().size()).isEqualTo(1);
assertThat(diagnostics[diagnostics.length - 1].getContactedRegionNames().iterator().next()).isEqualTo(SECOND_REGION_NAME);
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
ClientSideRequestStatistics[] clientStats =
diagnostics[0]
diagnostics[diagnostics.length - 1]
.getFeedResponseDiagnostics()
.getClientSideRequestStatistics()
.toArray(new ClientSideRequestStatistics[0]);
Expand Down Expand Up @@ -4067,10 +4079,14 @@ public Object[][] testConfigs_readAllAfterCreation() {

for (int i = start; i < diagnostics.length; i++) {
CosmosDiagnostics currentDiagnostics = diagnostics[i];
assertThat(currentDiagnostics.getFeedResponseDiagnostics()).isNotNull();
assertThat(currentDiagnostics.getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(currentDiagnostics.getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
assertThat(currentDiagnostics.getFeedResponseDiagnostics().getClientSideRequestStatistics().size()).isGreaterThanOrEqualTo(1);

if (i != diagnostics.length - 1) {
assertThat(currentDiagnostics.getFeedResponseDiagnostics()).isNull();
} else {
assertThat(currentDiagnostics.getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(currentDiagnostics.getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
assertThat(currentDiagnostics.getFeedResponseDiagnostics().getClientSideRequestStatistics().size()).isGreaterThanOrEqualTo(1);
}
}
}
};
Expand Down Expand Up @@ -4579,14 +4595,14 @@ public Object[][] testConfigs_readAllAfterCreation() {
CosmosDiagnostics[] diagnostics = ctx.getDiagnostics().toArray(new CosmosDiagnostics[0]);

// Diagnostics of query attempt in first region not even available yet
assertThat(diagnostics.length).isEqualTo(2);
assertThat(diagnostics[1].getContactedRegionNames().size()).isEqualTo(1);
assertThat(diagnostics[1].getContactedRegionNames().iterator().next()).isEqualTo(SECOND_REGION_NAME);
assertThat(diagnostics[1].getFeedResponseDiagnostics()).isNotNull();
assertThat(diagnostics[1].getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(diagnostics[1].getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
assertThat(diagnostics.length).isGreaterThanOrEqualTo(2);
assertThat(diagnostics[diagnostics.length - 1].getContactedRegionNames().size()).isEqualTo(1);
assertThat(diagnostics[diagnostics.length - 1].getContactedRegionNames().iterator().next()).isEqualTo(SECOND_REGION_NAME);
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics().getQueryMetricsMap()).isNotNull();
assertThat(diagnostics[diagnostics.length - 1].getFeedResponseDiagnostics().getClientSideRequestStatistics()).isNotNull();
ClientSideRequestStatistics[] clientStats =
diagnostics[1]
diagnostics[diagnostics.length - 1]
.getFeedResponseDiagnostics()
.getClientSideRequestStatistics()
.toArray(new ClientSideRequestStatistics[0]);
Expand Down
Loading
Loading