Add support for S3 request signing

CHANGELOG.md

@@ -29,6 +29,14 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 
 ### Highlights
 
+- Support for S3 remote request signing has been added, allowing Polaris to work with S3-compatible object storage systems.
+  *Remote signing is currently experimental and not enabled by default*. In particular, RBAC checks are currently not 
+  production-ready. One new table privilege was introduced: `TABLE_REMOTE_SIGN`. To enable remote signing:
+  1. Set the system-wide property `REMOTE_SIGNING_ENABLED` or the catalog-level `polaris.request-signing.enabled`
+     property to `true`.
+  2. Grant the `TABLE_REMOTE_SIGN` privilege to a catalog role. The role must also be granted the `TABLE_READ_DATA` 
+     and `TABLE_WRITE_DATA` privileges.
+
 ### Upgrade notes
 
 ### Breaking changes

LICENSE

@@ -217,6 +217,7 @@ This product includes code from Apache Iceberg.
 
 * spec/iceberg-rest-catalog-open-api.yaml
 * spec/polaris-catalog-apis/oauth-tokens-api.yaml
+* spec/s3-sign/iceberg-s3-signer-open-api.yaml
 * integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java
 * runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
 * runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/CatalogHandlerUtils.java

README.md

@@ -51,6 +51,7 @@ Apache Polaris is organized into the following modules:
   - `polaris-api-management-model` - The Polaris management model
   - `polaris-api-management-service` - The Polaris management service
   - `polaris-api-iceberg-service` - The Iceberg REST service
+  - `polaris-api-s3-sign-service` - The Iceberg REST service for S3 remote signing
 - Runtime modules:
   - `polaris-runtime-service` - The runtime components of the Polaris server
   - `polaris-runtime-defaults` - The runtime configuration defaults

api/README.md

@@ -33,6 +33,8 @@ This directory contains the API modules for Apache Polaris.
   Iceberg REST API.
 - [`polaris-api-catalog-service`](polaris-catalog-service): contains the service classes for the Polaris
   native Catalog REST API.
+- [`polaris-api-s3-sign-service`](s3-sign-service): contains the model and service classes 
+  for the S3 remote signing REST API.
 
 The classes in these modules are generated from the OpenAPI specification files in the
 [`spec`](../spec) directory.

api/s3-sign-service/build.gradle.kts

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
+
+plugins {
+  alias(libs.plugins.openapi.generator)
+  id("polaris-client")
+  alias(libs.plugins.jandex)
+}
+
+dependencies {
+  implementation(project(":polaris-core"))
+
+  implementation(platform(libs.iceberg.bom))
+  implementation("org.apache.iceberg:iceberg-api")
+  implementation("org.apache.iceberg:iceberg-core")
+  implementation("org.apache.iceberg:iceberg-aws")
+
+  implementation(libs.jakarta.annotation.api)
+  implementation(libs.jakarta.inject.api)
+  implementation(libs.jakarta.validation.api)
+  implementation(libs.swagger.annotations)
+
+  implementation(libs.jakarta.servlet.api)
+  implementation(libs.jakarta.ws.rs.api)
+
+  implementation(platform(libs.micrometer.bom))
+  implementation("io.micrometer:micrometer-core")
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-annotations")
+  implementation("com.fasterxml.jackson.core:jackson-core")
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.microprofile.fault.tolerance.api)
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
+}
+
+val rootDir = rootProject.layout.projectDirectory
+val specsDir = rootDir.dir("spec")
+val templatesDir = rootDir.dir("server-templates")
+// Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the
+// whole directory to its task output, but 'generated/' is not exclusive to that task and in turn
+// breaks Gradle's caching.
+val generatedDir = project.layout.buildDirectory.dir("generated-openapi")
+val generatedOpenApiSrcDir = project.layout.buildDirectory.dir("generated-openapi/src/main/java")
+
+openApiGenerate {
+  // The OpenAPI generator does NOT resolve relative paths correctly against the Gradle project
+  // directory
+  inputSpec = provider { specsDir.file("s3-sign/polaris-s3-sign-service.yaml").asFile.absolutePath }
+  generatorName = "jaxrs-resteasy"
+  outputDir = provider { generatedDir.get().asFile.absolutePath }
+  apiPackage = "org.apache.polaris.service.s3.sign.api"
+  ignoreFileOverride.set(provider { rootDir.file(".openapi-generator-ignore").asFile.absolutePath })
+  templateDir.set(provider { templatesDir.asFile.absolutePath })
+  removeOperationIdPrefix.set(true)
+  globalProperties.put("apis", "S3SignerApi")
+  globalProperties.put("models", "false")
+  globalProperties.put("apiDocs", "false")
+  globalProperties.put("modelTests", "false")
+  configOptions.put("resourceName", "catalog")
+  configOptions.put("useTags", "true")
+  configOptions.put("useBeanValidation", "false")
+  configOptions.put("sourceFolder", "src/main/java")
+  configOptions.put("useJakartaEe", "true")
+  configOptions.put("hideGenerationTimestamp", "true")
+  additionalProperties.put("apiNamePrefix", "IcebergRest")
+  additionalProperties.put("apiNameSuffix", "")
+  additionalProperties.put("metricsPrefix", "polaris")
+  serverVariables.put("basePath", "api/s3-sign")
+  modelNameMappings = mapOf("S3SignRequest" to "PolarisS3SignRequest")
+  typeMappings =
+    mapOf("S3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest")
+  importMappings =
+    mapOf(
+      "IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse",
+      "PolarisS3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest",
+      "SignS3Request200Response" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse",
+    )
+}
+
+listOf("sourcesJar", "compileJava", "processResources").forEach { task ->
+  tasks.named(task) { dependsOn("openApiGenerate") }
+}
+
+sourceSets { main { java { srcDir(generatedOpenApiSrcDir) } } }
+
+tasks.named<GenerateTask>("openApiGenerate") {
+  inputs.dir(templatesDir)
+  inputs.dir(specsDir)
+  actions.addFirst { delete { delete(generatedDir) } }
+}
+
+tasks.named("javadoc") { dependsOn("jandex") }

api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.s3.sign.model;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import jakarta.annotation.Nullable;
+import org.apache.iceberg.aws.s3.signer.S3SignRequest;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+/**
+ * Request for S3 signing.
+ *
+ * <p>Copy of {@link S3SignRequest}, because the original does not have Jackson annotations.
+ */
+@PolarisImmutable
+@JsonDeserialize(as = ImmutablePolarisS3SignRequest.class)
+@JsonSerialize(as = ImmutablePolarisS3SignRequest.class)
+@SuppressWarnings("immutables:subtype")
+public interface PolarisS3SignRequest extends S3SignRequest {
+
+  @Value.Default
+  @Nullable // Replace javax.annotation.Nullable from S3SignRequest with jakarta.annotation.Nullable
+  @Override
+  default String body() {
+    return null;
+  }
+
+  default boolean write() {
+    return method().equalsIgnoreCase("PUT")
+        || method().equalsIgnoreCase("POST")
+        || method().equalsIgnoreCase("DELETE")
+        || method().equalsIgnoreCase("PATCH");
+  }
+}

api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.s3.sign.model;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import org.apache.iceberg.aws.s3.signer.S3SignResponse;
+import org.apache.polaris.immutables.PolarisImmutable;
+
+/**
+ * Response for S3 signing requests.
+ *
+ * <p>Copy of {@link S3SignResponse}, because the original does not have Jackson annotations.
+ */
+@PolarisImmutable
+@JsonDeserialize(as = ImmutablePolarisS3SignResponse.class)
+@JsonSerialize(as = ImmutablePolarisS3SignResponse.class)
+@SuppressWarnings("immutables:subtype")
+public interface PolarisS3SignResponse extends S3SignResponse {}

bom/build.gradle.kts

@@ -28,6 +28,7 @@ dependencies {
     api(project(":polaris-api-iceberg-service"))
     api(project(":polaris-api-management-model"))
     api(project(":polaris-api-management-service"))
+    api(project(":polaris-api-s3-sign-service"))
 
     api(project(":polaris-container-spec-helper"))
     api(project(":polaris-minio-testcontainer"))

build.gradle.kts

@@ -152,6 +152,7 @@ tasks.register<Exec>("regeneratePythonClient") {
   dependsOn(":polaris-api-management-service:processResources")
   dependsOn(":polaris-api-catalog-service:processResources")
   dependsOn(":polaris-api-management-model:processResources")
+  dependsOn(":polaris-api-s3-sign-service:processResources")
 }
 
 // Pass environment variables:

gradle/projects.main.properties

@@ -24,6 +24,7 @@ polaris-api-iceberg-service=api/iceberg-service
 polaris-api-management-model=api/management-model
 polaris-api-management-service=api/management-service
 polaris-api-catalog-service=api/polaris-catalog-service
+polaris-api-s3-sign-service=api/s3-sign-service
 polaris-runtime-defaults=runtime/defaults
 polaris-runtime-service=runtime/service
 polaris-server=runtime/server