Spanner Persistence Backend for Polaris

.gitignore

@@ -100,3 +100,6 @@ hs_err_pid*
 
 # macOS
 *.DS_Store
+
+# VSCode
+.vscode/settings.json

gradle/libs.versions.toml

@@ -49,7 +49,7 @@ commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.18.0
 commons-text = { module = "org.apache.commons:commons-text", version = "1.14.0" }
 eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.7" }
 errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.41.0" }
-google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.55.0" }
+google-cloud-libraries-bom = { module = "com.google.cloud:libraries-bom", version = "26.64.0" }
 guava = { module = "com.google.guava:guava", version = "33.4.8-jre" }
 h2 = { module = "com.h2database:h2", version = "2.3.232" }
 dnsjava = { module = "dnsjava:dnsjava", version = "3.6.3" }

gradle/projects.main.properties

@@ -34,6 +34,7 @@ polaris-runtime-common=runtime/common
 polaris-runtime-test-common=runtime/test-common
 polaris-eclipselink=persistence/eclipselink
 polaris-relational-jdbc=persistence/relational-jdbc
+polaris-google-cloud-spanner=persistence/google-cloud-spanner
 polaris-tests=integration-tests
 aggregated-license-report=aggregated-license-report
 polaris-immutables=tools/immutables

persistence/google-cloud-spanner/build.gradle.kts

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  id("polaris-server")
+  alias(libs.plugins.jandex)
+}
+
+dependencies {
+  implementation(project(":polaris-core"))
+  implementation(libs.slf4j.api)
+  implementation(libs.guava)
+
+  implementation(platform(libs.google.cloud.libraries.bom))
+  implementation("com.google.cloud:google-cloud-spanner")
+
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+  compileOnly(libs.jakarta.inject.api)
+
+  compileOnly(libs.smallrye.common.annotation) // @Identifier
+  compileOnly(libs.smallrye.config.core) // @ConfigMapping
+
+  testImplementation(libs.mockito.junit.jupiter)
+  testImplementation(libs.h2)
+  testImplementation(testFixtures(project(":polaris-core")))
+}

persistence/google-cloud-spanner/src/main/java/org/apache/polaris/persistence/relational/spanner/DatabaseAdminClientSupplier.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.spanner;
+
+import com.google.cloud.spanner.DatabaseAdminClient;
+import java.util.function.Supplier;
+
+public interface DatabaseAdminClientSupplier extends Supplier<DatabaseAdminClient> {}

persistence/google-cloud-spanner/src/main/java/org/apache/polaris/persistence/relational/spanner/DatabaseClientSupplier.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.spanner;
+
+import com.google.cloud.spanner.DatabaseClient;
+import java.util.function.Supplier;
+
+public interface DatabaseClientSupplier extends Supplier<DatabaseClient> {}

persistence/google-cloud-spanner/src/main/java/org/apache/polaris/persistence/relational/spanner/GoogleCloudSpannerConfiguration.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.spanner;
+
+import io.smallrye.config.ConfigMapping;
+import java.util.Optional;
+
+@ConfigMapping(prefix = "polaris.persistence.spanner")
+public interface GoogleCloudSpannerConfiguration {
+
+  public Optional<String> quotaProjectId();
+
+  public Optional<String> projectId();
+
+  public Optional<String> instanceId();
+
+  public Optional<String> databaseId();
+
+  public Optional<String> emulatorHost();
+}

persistence/google-cloud-spanner/src/main/java/org/apache/polaris/persistence/relational/spanner/GoogleCloudSpannerDatabaseClientLifecycleManager.java

@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.spanner;
+
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.DatabaseId;
+import com.google.cloud.spanner.Dialect;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerException;
+import com.google.common.collect.ImmutableList;
+import jakarta.annotation.PostConstruct;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Produces;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Consumer;
+import org.apache.polaris.core.context.RealmContext;
+import org.apache.polaris.core.persistence.bootstrap.SchemaOptions;
+import org.apache.polaris.persistence.relational.spanner.model.Realm;
+import org.apache.polaris.persistence.relational.spanner.util.SpannerUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@ApplicationScoped
+public class GoogleCloudSpannerDatabaseClientLifecycleManager {
+
+  private static Logger LOGGER =
+      LoggerFactory.getLogger(GoogleCloudSpannerDatabaseClientLifecycleManager.class);
+
+  protected final GoogleCloudSpannerConfiguration spannerConfiguration;
+
+  public GoogleCloudSpannerDatabaseClientLifecycleManager(
+      GoogleCloudSpannerConfiguration spannerConfiguration) {
+    this.spannerConfiguration = spannerConfiguration;
+  }
+
+  protected Spanner spanner;
+  protected DatabaseId databaseId;
+
+  @PostConstruct
+  protected void init() {
+    spanner = SpannerUtil.spannerFromConfiguration(spannerConfiguration);
+    databaseId = SpannerUtil.databaseFromConfiguration(spannerConfiguration);
+  }
+
+  protected List<String> getSpannerDatabaseDdl(SchemaOptions options) {
+    final InputStream schemaStream;
+    if (options.schemaFile() != null) {
+      try {
+        schemaStream = new FileInputStream(options.schemaFile());
+      } catch (IOException e) {
+        throw new IllegalArgumentException("Unable to load file " + options.schemaFile(), e);
+      }
+    } else {
+      if (options.schemaVersion() == null || options.schemaVersion() == 1) {
+        schemaStream =
+            getClass().getResourceAsStream("/org/apache/polaris/persistence/spanner/schema-v1.sql");
+      } else {
+        throw new IllegalArgumentException("Unknown schema version " + options.schemaVersion());
+      }
+    }
+    try (schemaStream) {
+      String schema = new String(schemaStream.readAllBytes(), Charset.forName("UTF-8"));
+      List<String> lines = new ArrayList<>();
+      for (String s : schema.split("\n")) {
+        s = s.trim();
+        if (s.startsWith("--") || s.length() == 0) {
+          continue;
+        }
+        lines.add(s);
+      }
+      return List.of(String.join(" ", lines).split(";"));
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to retrieve DDL statements", e);
+    }
+  }
+
+  @Produces
+  public SchemaInitializer getSchemaInitializer() {
+    return (options) -> {
+      List<String> ddlStatements = getSpannerDatabaseDdl(options);
+      LOGGER.info(
+          "Attempting to initialize Spanner database DDL with {} statements,",
+          ddlStatements.size());
+      DatabaseAdminClient client = spanner.getDatabaseAdminClient();
+      Database dbInfo =
+          client.newDatabaseBuilder(databaseId).setDialect(Dialect.GOOGLE_STANDARD_SQL).build();
+      try {
+        spanner.getDatabaseAdminClient().updateDatabaseDdl(dbInfo, ddlStatements, null).get();
+        LOGGER.info("Successfully applied DDL update.");
+      } catch (InterruptedException | ExecutionException e) {
+        LOGGER.error("Unable to update Spanner DDL.", e);
+        throw new RuntimeException(
+            "Unable to update Spanner DDL. Please disable this option for this database configuration.",
+            e);
+      }
+    };
+  }
+
+  @Produces
+  public Consumer<RealmContext> getRealmInitializer() {
+    return (realmContext) -> {
+      try {
+        spanner
+            .getDatabaseClient(databaseId)
+            .write(ImmutableList.of(Realm.upsert(realmContext.getRealmIdentifier())));
+      } catch (SpannerException e) {
+        LOGGER.error("Unable to initialize realm " + realmContext.getRealmIdentifier(), e);
+      }
+    };
+  }
+
+  @Produces
+  public DatabaseClientSupplier getDatabaseClientSupplier() {
+    return () -> spanner.getDatabaseClient(databaseId);
+  }
+
+  @Produces
+  public DatabaseAdminClientSupplier getDatabaseAdminClientSupplier() {
+    return () -> spanner.getDatabaseAdminClient();
+  }
+}

persistence/google-cloud-spanner/src/main/java/org/apache/polaris/persistence/relational/spanner/RealmInitializer.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.spanner;
+
+import java.util.function.Consumer;
+import org.apache.polaris.core.context.RealmContext;
+
+public interface RealmInitializer extends Consumer<RealmContext> {}

persistence/google-cloud-spanner/src/main/java/org/apache/polaris/persistence/relational/spanner/SchemaInitializer.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.spanner;
+
+import java.util.function.Consumer;
+import org.apache.polaris.core.persistence.bootstrap.SchemaOptions;
+
+public interface SchemaInitializer extends Consumer<SchemaOptions> {}

persistence/google-cloud-spanner/src/main/java/org/apache/polaris/persistence/relational/spanner/model/Realm.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.persistence.relational.spanner.model;
+
+import com.google.cloud.spanner.Key;
+import com.google.cloud.spanner.Mutation;
+
+public final class Realm {
+
+  public static String TABLE_NAME = "Realms";
+
+  public static Mutation upsert(String realmId) {
+    return Mutation.newInsertOrUpdateBuilder(TABLE_NAME).set("RealmId").to(realmId).build();
+  }
+
+  public static Mutation delete(String realmId) {
+    return Mutation.delete(TABLE_NAME, Key.of(realmId));
+  }
+}