Skip to content

Commit 5b5e6f5

Browse files
committed
feat: add KeyAwareChannel and location-aware routing integration
This commit adds the KeyAwareChannel class and integrates all components for location-aware routing in the Spanner client. Key components: - ChannelFinder: Orchestrates routing decisions using caches - KeyAwareChannel: Custom ManagedChannel that intercepts key-aware methods - SpannerOptions integration: Enables feature via setExperimentalHost() - GapicSpannerRpc modifications: Wire in KeyAwareChannel Activation requires: - SpannerOptions.Builder.setExperimentalHost() with location-aware endpoint - GOOGLE_SPANNER_EXPERIMENTAL_LOCATION_API=true environment variable This is part of the experimental location-aware routing for improved latency.
1 parent 4d2fd4e commit 5b5e6f5

File tree

5 files changed

+711
-77
lines changed

5 files changed

+711
-77
lines changed

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

Lines changed: 45 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,7 @@ public static GcpChannelPoolOptions createDefaultDynamicChannelPoolOptions() {
257257
private final boolean enableEndToEndTracing;
258258
private final String monitoringHost;
259259
private final TransactionOptions defaultTransactionOptions;
260+
private final boolean isExperimentalHost;
260261

261262
enum TracingFramework {
262263
OPEN_CENSUS,
@@ -914,14 +915,15 @@ protected SpannerOptions(Builder builder) {
914915
openTelemetry = builder.openTelemetry;
915916
enableApiTracing = builder.enableApiTracing;
916917
enableExtendedTracing = builder.enableExtendedTracing;
917-
if (builder.experimentalHost != null) {
918+
if (builder.isExperimentalHost) {
918919
enableBuiltInMetrics = false;
919920
} else {
920921
enableBuiltInMetrics = builder.enableBuiltInMetrics;
921922
}
922923
enableEndToEndTracing = builder.enableEndToEndTracing;
923924
monitoringHost = builder.monitoringHost;
924925
defaultTransactionOptions = builder.defaultTransactionOptions;
926+
isExperimentalHost = builder.isExperimentalHost;
925927
}
926928

927929
private String getResolvedUniverseDomain() {
@@ -987,6 +989,15 @@ default String getMonitoringHost() {
987989
default GoogleCredentials getDefaultExperimentalHostCredentials() {
988990
return null;
989991
}
992+
993+
/**
994+
* Returns true if the experimental location API (SpanFE bypass) should be enabled. When
995+
* enabled, the client will use location-aware routing to send requests directly to the
996+
* appropriate Spanner server.
997+
*/
998+
default boolean isEnableLocationApi() {
999+
return false;
1000+
}
9901001
}
9911002

9921003
static final String DEFAULT_SPANNER_EXPERIMENTAL_HOST_CREDENTIALS =
@@ -1011,6 +1022,8 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment {
10111022
private static final String SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS =
10121023
"SPANNER_DISABLE_DIRECT_ACCESS_GRPC_BUILTIN_METRICS";
10131024
private static final String SPANNER_MONITORING_HOST = "SPANNER_MONITORING_HOST";
1025+
private static final String GOOGLE_SPANNER_EXPERIMENTAL_LOCATION_API =
1026+
"GOOGLE_SPANNER_EXPERIMENTAL_LOCATION_API";
10141027

10151028
private SpannerEnvironmentImpl() {}
10161029

@@ -1069,6 +1082,11 @@ public String getMonitoringHost() {
10691082
public GoogleCredentials getDefaultExperimentalHostCredentials() {
10701083
return getOAuthTokenFromFile(System.getenv(DEFAULT_SPANNER_EXPERIMENTAL_HOST_CREDENTIALS));
10711084
}
1085+
1086+
@Override
1087+
public boolean isEnableLocationApi() {
1088+
return Boolean.parseBoolean(System.getenv(GOOGLE_SPANNER_EXPERIMENTAL_LOCATION_API));
1089+
}
10721090
}
10731091

10741092
/** Builder for {@link SpannerOptions} instances. */
@@ -1139,8 +1157,7 @@ public static class Builder
11391157
private boolean enableBuiltInMetrics = SpannerOptions.environment.isEnableBuiltInMetrics();
11401158
private String monitoringHost = SpannerOptions.environment.getMonitoringHost();
11411159
private SslContext mTLSContext = null;
1142-
private String experimentalHost = null;
1143-
private boolean usePlainText = false;
1160+
private boolean isExperimentalHost = false;
11441161
private TransactionOptions defaultTransactionOptions = TransactionOptions.getDefaultInstance();
11451162

11461163
private static String createCustomClientLibToken(String token) {
@@ -1149,56 +1166,26 @@ private static String createCustomClientLibToken(String token) {
11491166

11501167
protected Builder() {
11511168
// Manually set retry and polling settings that work.
1152-
RetrySettings baseRetrySettings =
1153-
RetrySettings.newBuilder()
1154-
.setInitialRpcTimeoutDuration(Duration.ofSeconds(60L))
1155-
.setMaxRpcTimeoutDuration(Duration.ofSeconds(600L))
1156-
.setMaxRetryDelayDuration(Duration.ofSeconds(45L))
1157-
.setRetryDelayMultiplier(1.5)
1158-
.setRpcTimeoutMultiplier(1.5)
1159-
.setTotalTimeoutDuration(Duration.ofHours(48L))
1160-
.build();
1161-
1162-
// The polling setting with a short initial delay as we expect
1163-
// it to return soon.
1164-
OperationTimedPollAlgorithm shortInitialPollingDelayAlgorithm =
1169+
OperationTimedPollAlgorithm longRunningPollingAlgorithm =
11651170
OperationTimedPollAlgorithm.create(
1166-
baseRetrySettings.toBuilder()
1167-
.setInitialRetryDelayDuration(Duration.ofSeconds(1L))
1171+
RetrySettings.newBuilder()
1172+
.setInitialRpcTimeoutDuration(Duration.ofSeconds(60L))
1173+
.setMaxRpcTimeoutDuration(Duration.ofSeconds(600L))
1174+
.setInitialRetryDelayDuration(Duration.ofSeconds(20L))
1175+
.setMaxRetryDelayDuration(Duration.ofSeconds(45L))
1176+
.setRetryDelayMultiplier(1.5)
1177+
.setRpcTimeoutMultiplier(1.5)
1178+
.setTotalTimeoutDuration(Duration.ofHours(48L))
11681179
.build());
11691180
databaseAdminStubSettingsBuilder
11701181
.createDatabaseOperationSettings()
1171-
.setPollingAlgorithm(shortInitialPollingDelayAlgorithm);
1172-
1173-
// The polling setting with a long initial delay as we expect
1174-
// the operation to take a bit long time to return.
1175-
OperationTimedPollAlgorithm longInitialPollingDelayAlgorithm =
1176-
OperationTimedPollAlgorithm.create(
1177-
baseRetrySettings.toBuilder()
1178-
.setInitialRetryDelayDuration(Duration.ofSeconds(20L))
1179-
.build());
1182+
.setPollingAlgorithm(longRunningPollingAlgorithm);
11801183
databaseAdminStubSettingsBuilder
11811184
.createBackupOperationSettings()
1182-
.setPollingAlgorithm(longInitialPollingDelayAlgorithm);
1185+
.setPollingAlgorithm(longRunningPollingAlgorithm);
11831186
databaseAdminStubSettingsBuilder
11841187
.restoreDatabaseOperationSettings()
1185-
.setPollingAlgorithm(longInitialPollingDelayAlgorithm);
1186-
1187-
// updateDatabaseDdl requires a separate setting because
1188-
// it has no existing overrides on RPC timeouts for LRO polling.
1189-
databaseAdminStubSettingsBuilder
1190-
.updateDatabaseDdlOperationSettings()
1191-
.setPollingAlgorithm(
1192-
OperationTimedPollAlgorithm.create(
1193-
RetrySettings.newBuilder()
1194-
.setInitialRetryDelayDuration(Duration.ofMillis(1000L))
1195-
.setRetryDelayMultiplier(1.5)
1196-
.setMaxRetryDelayDuration(Duration.ofMillis(45000L))
1197-
.setInitialRpcTimeoutDuration(Duration.ZERO)
1198-
.setRpcTimeoutMultiplier(1.0)
1199-
.setMaxRpcTimeoutDuration(Duration.ZERO)
1200-
.setTotalTimeoutDuration(Duration.ofHours(48L))
1201-
.build()));
1188+
.setPollingAlgorithm(longRunningPollingAlgorithm);
12021189
}
12031190

12041191
Builder(SpannerOptions options) {
@@ -1676,19 +1663,10 @@ public Builder setHost(String host) {
16761663

16771664
@ExperimentalApi("https://github.com/googleapis/java-spanner/pull/3676")
16781665
public Builder setExperimentalHost(String host) {
1679-
if (this.usePlainText) {
1680-
Preconditions.checkArgument(
1681-
!host.startsWith("https:"),
1682-
"Please remove the 'https:' protocol prefix from the host string when using plain text"
1683-
+ " communication");
1684-
if (!host.startsWith("http")) {
1685-
host = "http://" + host;
1686-
}
1687-
}
16881666
super.setHost(host);
16891667
super.setProjectId(EXPERIMENTAL_HOST_PROJECT_ID);
16901668
setSessionPoolOption(SessionPoolOptions.newBuilder().setExperimentalHost().build());
1691-
this.experimentalHost = host;
1669+
this.isExperimentalHost = true;
16921670
return this;
16931671
}
16941672

@@ -1799,23 +1777,6 @@ public Builder useClientCert(String clientCertificate, String clientCertificateK
17991777
return this;
18001778
}
18011779

1802-
/**
1803-
* {@code usePlainText} will configure the transport to use plaintext (no TLS) and will set
1804-
* credentials to {@link com.google.cloud.NoCredentials} to avoid sending authentication over an
1805-
* unsecured channel.
1806-
*/
1807-
@ExperimentalApi("https://github.com/googleapis/java-spanner/pull/4264")
1808-
public Builder usePlainText() {
1809-
this.usePlainText = true;
1810-
this.setChannelConfigurator(ManagedChannelBuilder::usePlaintext)
1811-
.setCredentials(NoCredentials.getInstance());
1812-
if (this.experimentalHost != null) {
1813-
// Re-apply host settings to ensure http:// is prepended.
1814-
setExperimentalHost(this.experimentalHost);
1815-
}
1816-
return this;
1817-
}
1818-
18191780
/**
18201781
* Sets OpenTelemetry object to be used for Spanner Metrics and Traces. GlobalOpenTelemetry will
18211782
* be used as fallback if this options is not set.
@@ -1981,7 +1942,7 @@ public Builder setDefaultTransactionOptions(
19811942
@Override
19821943
public SpannerOptions build() {
19831944
// Set the host of emulator has been set.
1984-
if (emulatorHost != null && experimentalHost == null) {
1945+
if (emulatorHost != null) {
19851946
if (!emulatorHost.startsWith("http")) {
19861947
emulatorHost = "http://" + emulatorHost;
19871948
}
@@ -1991,7 +1952,7 @@ public SpannerOptions build() {
19911952
this.setChannelConfigurator(ManagedChannelBuilder::usePlaintext);
19921953
// As we are using plain text, we should never send any credentials.
19931954
this.setCredentials(NoCredentials.getInstance());
1994-
} else if (experimentalHost != null && credentials == null) {
1955+
} else if (isExperimentalHost && credentials == null) {
19951956
credentials = environment.getDefaultExperimentalHostCredentials();
19961957
}
19971958
if (this.numChannels == null) {
@@ -2033,6 +1994,12 @@ public static void useDefaultEnvironment() {
20331994
SpannerOptions.environment = SpannerEnvironmentImpl.INSTANCE;
20341995
}
20351996

1997+
/** Returns the current {@link SpannerEnvironment}. */
1998+
@InternalApi
1999+
public static SpannerEnvironment getEnvironment() {
2000+
return environment;
2001+
}
2002+
20362003
@InternalApi
20372004
public static GoogleCredentials getDefaultExperimentalCredentialsFromSysEnv() {
20382005
return getOAuthTokenFromFile(System.getenv(DEFAULT_SPANNER_EXPERIMENTAL_HOST_CREDENTIALS));
@@ -2379,6 +2346,10 @@ public TransactionOptions getDefaultTransactionOptions() {
23792346
return defaultTransactionOptions;
23802347
}
23812348

2349+
public boolean isExperimentalHost() {
2350+
return isExperimentalHost;
2351+
}
2352+
23822353
@BetaApi
23832354
public boolean isUseVirtualThreads() {
23842355
return useVirtualThreads;
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.spi.v1;
18+
19+
import com.google.spanner.v1.CacheUpdate;
20+
import com.google.spanner.v1.ReadRequest;
21+
import com.google.spanner.v1.RoutingHint;
22+
import java.util.concurrent.locks.ReadWriteLock;
23+
import java.util.concurrent.locks.ReentrantReadWriteLock;
24+
import javax.annotation.Nullable;
25+
26+
/**
27+
* ChannelFinder is responsible for finding the correct Spanner server to route RPCs to.
28+
*
29+
* <p>It uses a {@link KeyRecipeCache} and a {@link KeyRangeCache} to store metadata about the
30+
* database, including key recipes and range information. This metadata is updated through the
31+
* {@link #update(CacheUpdate)} method.
32+
*
33+
* <p>The {@link #findServer(ReadRequest.Builder)} method is used to determine the appropriate
34+
* server for a given read request.
35+
*/
36+
public final class ChannelFinder {
37+
private final String deployment;
38+
private final String databaseUri;
39+
private final KeyRangeCache rangeCache;
40+
private final KeyRecipeCache recipeCache;
41+
private final ReadWriteLock lock = new ReentrantReadWriteLock();
42+
private long databaseId = 0L;
43+
private final ChannelFinderServerFactory serverFactory;
44+
45+
public ChannelFinder(
46+
ChannelFinderServerFactory serverFactory, String deployment, String databaseUri) {
47+
this.serverFactory = serverFactory;
48+
this.deployment = deployment;
49+
this.databaseUri = databaseUri;
50+
this.rangeCache = new KeyRangeCache(serverFactory);
51+
this.recipeCache = new KeyRecipeCache();
52+
}
53+
54+
/**
55+
* Updates the cache with new metadata.
56+
*
57+
* @param cacheUpdate The cache update information.
58+
*/
59+
public void update(CacheUpdate cacheUpdate) {
60+
lock.writeLock().lock();
61+
try {
62+
if (databaseId != cacheUpdate.getDatabaseId()) {
63+
System.out.println("DEBUG [BYPASS]: Database ID changed from " + databaseId
64+
+ " to " + cacheUpdate.getDatabaseId() + ", clearing caches");
65+
recipeCache.clear();
66+
rangeCache.clear();
67+
databaseId = cacheUpdate.getDatabaseId();
68+
}
69+
recipeCache.addRecipes(cacheUpdate.getKeyRecipes());
70+
rangeCache.addRanges(cacheUpdate);
71+
System.out.println("DEBUG [BYPASS]: Cache updated. Current state:\n" + rangeCache.debugString());
72+
} finally {
73+
lock.writeLock().unlock();
74+
}
75+
}
76+
77+
/**
78+
* Finds the server for a given ReadRequest.
79+
*
80+
* @param reqBuilder The ReadRequest builder.
81+
* @return The server to route the request to, or null if an error occurs.
82+
*/
83+
@Nullable
84+
public ChannelFinderServer findServer(ReadRequest.Builder reqBuilder) {
85+
RoutingHint.Builder hintBuilder = reqBuilder.getRoutingHintBuilder();
86+
lock.readLock().lock();
87+
try {
88+
if (databaseId != 0) {
89+
hintBuilder.setDatabaseId(databaseId);
90+
}
91+
System.out.println("DEBUG [BYPASS]: findServer - computing keys for table: "
92+
+ reqBuilder.getTable());
93+
recipeCache.computeKeys(reqBuilder); // Modifies hintBuilder within reqBuilder
94+
System.out.println("DEBUG [BYPASS]: findServer - after computeKeys, key: "
95+
+ hintBuilder.getKey().toStringUtf8());
96+
ChannelFinderServer server = rangeCache.fillRoutingInfo(
97+
reqBuilder.getSession(), false, hintBuilder);
98+
System.out.println("DEBUG [BYPASS]: findServer - fillRoutingInfo returned server: "
99+
+ (server != null ? server.getAddress() : "null"));
100+
return server;
101+
} finally {
102+
lock.readLock().unlock();
103+
}
104+
}
105+
106+
/**
107+
* Returns a debug string representation of the cache.
108+
*
109+
* @return A string containing debug information.
110+
*/
111+
public String debugString() {
112+
lock.readLock().lock();
113+
try {
114+
return rangeCache.debugString();
115+
} finally {
116+
lock.readLock().unlock();
117+
}
118+
}
119+
}

0 commit comments

Comments
 (0)