Skip to content

Commit 7fefc85

Browse files
adutraadutra
committed
Add support for S3 request signing
Fixes #32.
1 parent 42225b3 commit 7fefc85

File tree

64 files changed

+2397
-316
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+2397
-316
lines changed

‎CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
2929

3030
### Highlights
3131

32+
- Support for S3 remote request signing has been added, allowing Polaris to work with S3-compatible object storage systems.
33+
*Remote signing is currently experimental and not enabled by default*. In particular, RBAC checks are currently not
34+
production-ready. One new table privilege was introduced: `TABLE_REMOTE_SIGN`. To enable remote signing:
35+
1. Set the system-wide property `REMOTE_SIGNING_ENABLED` or the catalog-level `polaris.request-signing.enabled`
36+
property to `true`.
37+
2. Grant the `TABLE_REMOTE_SIGN` privilege to a catalog role. The role must also be granted the `TABLE_READ_DATA`
38+
and `TABLE_WRITE_DATA` privileges.
39+
3240
### Upgrade notes
3341

3442
### Breaking changes

‎LICENSE

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ This product includes code from Apache Iceberg.
217217

218218
* spec/iceberg-rest-catalog-open-api.yaml
219219
* spec/polaris-catalog-apis/oauth-tokens-api.yaml
220+
* spec/s3-sign/iceberg-s3-signer-open-api.yaml
220221
* integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java
221222
* runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
222223
* runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/CatalogHandlerUtils.java

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ Apache Polaris is organized into the following modules:
5151
- `polaris-api-management-model` - The Polaris management model
5252
- `polaris-api-management-service` - The Polaris management service
5353
- `polaris-api-iceberg-service` - The Iceberg REST service
54+
- `polaris-api-s3-sign-service` - The Iceberg REST service for S3 remote signing
5455
- Runtime modules:
5556
- `polaris-runtime-service` - The runtime components of the Polaris server
5657
- `polaris-runtime-defaults` - The runtime configuration defaults

‎api/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ This directory contains the API modules for Apache Polaris.
3333
Iceberg REST API.
3434
- [`polaris-api-catalog-service`](polaris-catalog-service): contains the service classes for the Polaris
3535
native Catalog REST API.
36+
- [`polaris-api-s3-sign-service`](s3-sign-service): contains the model and service classes
37+
for the S3 remote signing REST API.
3638

3739
The classes in these modules are generated from the OpenAPI specification files in the
3840
[`spec`](../spec) directory.

‎api/s3-sign-service/build.gradle.kts

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
21+
22+
plugins {
23+
alias(libs.plugins.openapi.generator)
24+
id("polaris-client")
25+
alias(libs.plugins.jandex)
26+
}
27+
28+
dependencies {
29+
implementation(project(":polaris-core"))
30+
31+
implementation(platform(libs.iceberg.bom))
32+
implementation("org.apache.iceberg:iceberg-api")
33+
implementation("org.apache.iceberg:iceberg-core")
34+
implementation("org.apache.iceberg:iceberg-aws")
35+
36+
implementation(libs.jakarta.annotation.api)
37+
implementation(libs.jakarta.inject.api)
38+
implementation(libs.jakarta.validation.api)
39+
implementation(libs.swagger.annotations)
40+
41+
implementation(libs.jakarta.servlet.api)
42+
implementation(libs.jakarta.ws.rs.api)
43+
44+
implementation(platform(libs.micrometer.bom))
45+
implementation("io.micrometer:micrometer-core")
46+
47+
implementation(platform(libs.jackson.bom))
48+
implementation("com.fasterxml.jackson.core:jackson-annotations")
49+
implementation("com.fasterxml.jackson.core:jackson-core")
50+
implementation("com.fasterxml.jackson.core:jackson-databind")
51+
52+
compileOnly(libs.microprofile.fault.tolerance.api)
53+
54+
compileOnly(project(":polaris-immutables"))
55+
annotationProcessor(project(":polaris-immutables", configuration = "processor"))
56+
}
57+
58+
val rootDir = rootProject.layout.projectDirectory
59+
val specsDir = rootDir.dir("spec")
60+
val templatesDir = rootDir.dir("server-templates")
61+
// Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the
62+
// whole directory to its task output, but 'generated/' is not exclusive to that task and in turn
63+
// breaks Gradle's caching.
64+
val generatedDir = project.layout.buildDirectory.dir("generated-openapi")
65+
val generatedOpenApiSrcDir = project.layout.buildDirectory.dir("generated-openapi/src/main/java")
66+
67+
openApiGenerate {
68+
// The OpenAPI generator does NOT resolve relative paths correctly against the Gradle project
69+
// directory
70+
inputSpec = provider { specsDir.file("s3-sign/polaris-s3-sign-service.yaml").asFile.absolutePath }
71+
generatorName = "jaxrs-resteasy"
72+
outputDir = provider { generatedDir.get().asFile.absolutePath }
73+
apiPackage = "org.apache.polaris.service.s3.sign.api"
74+
ignoreFileOverride.set(provider { rootDir.file(".openapi-generator-ignore").asFile.absolutePath })
75+
templateDir.set(provider { templatesDir.asFile.absolutePath })
76+
removeOperationIdPrefix.set(true)
77+
globalProperties.put("apis", "S3SignerApi")
78+
globalProperties.put("models", "false")
79+
globalProperties.put("apiDocs", "false")
80+
globalProperties.put("modelTests", "false")
81+
configOptions.put("resourceName", "catalog")
82+
configOptions.put("useTags", "true")
83+
configOptions.put("useBeanValidation", "false")
84+
configOptions.put("sourceFolder", "src/main/java")
85+
configOptions.put("useJakartaEe", "true")
86+
configOptions.put("hideGenerationTimestamp", "true")
87+
additionalProperties.put("apiNamePrefix", "IcebergRest")
88+
additionalProperties.put("apiNameSuffix", "")
89+
additionalProperties.put("metricsPrefix", "polaris")
90+
serverVariables.put("basePath", "api/s3-sign")
91+
modelNameMappings = mapOf("S3SignRequest" to "PolarisS3SignRequest")
92+
typeMappings =
93+
mapOf("S3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest")
94+
importMappings =
95+
mapOf(
96+
"IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse",
97+
"PolarisS3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest",
98+
"SignS3Request200Response" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse",
99+
)
100+
}
101+
102+
listOf("sourcesJar", "compileJava", "processResources").forEach { task ->
103+
tasks.named(task) { dependsOn("openApiGenerate") }
104+
}
105+
106+
sourceSets { main { java { srcDir(generatedOpenApiSrcDir) } } }
107+
108+
tasks.named<GenerateTask>("openApiGenerate") {
109+
inputs.dir(templatesDir)
110+
inputs.dir(specsDir)
111+
actions.addFirst { delete { delete(generatedDir) } }
112+
}
113+
114+
tasks.named("javadoc") { dependsOn("jandex") }

‎api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.polaris.service.s3.sign.model;
21+
22+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
23+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
24+
import jakarta.annotation.Nullable;
25+
import org.apache.iceberg.aws.s3.signer.S3SignRequest;
26+
import org.apache.polaris.immutables.PolarisImmutable;
27+
import org.immutables.value.Value;
28+
29+
/**
30+
* Request for S3 signing.
31+
*
32+
* <p>Copy of {@link S3SignRequest}, because the original does not have Jackson annotations.
33+
*/
34+
@PolarisImmutable
35+
@JsonDeserialize(as = ImmutablePolarisS3SignRequest.class)
36+
@JsonSerialize(as = ImmutablePolarisS3SignRequest.class)
37+
@SuppressWarnings("immutables:subtype")
38+
public interface PolarisS3SignRequest extends S3SignRequest {
39+
40+
@Value.Default
41+
@Nullable // Replace javax.annotation.Nullable from S3SignRequest with jakarta.annotation.Nullable
42+
@Override
43+
default String body() {
44+
return null;
45+
}
46+
47+
default boolean write() {
48+
return method().equalsIgnoreCase("PUT")
49+
|| method().equalsIgnoreCase("POST")
50+
|| method().equalsIgnoreCase("DELETE")
51+
|| method().equalsIgnoreCase("PATCH");
52+
}
53+
}

‎api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.polaris.service.s3.sign.model;
21+
22+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
23+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
24+
import org.apache.iceberg.aws.s3.signer.S3SignResponse;
25+
import org.apache.polaris.immutables.PolarisImmutable;
26+
27+
/**
28+
* Response for S3 signing requests.
29+
*
30+
* <p>Copy of {@link S3SignResponse}, because the original does not have Jackson annotations.
31+
*/
32+
@PolarisImmutable
33+
@JsonDeserialize(as = ImmutablePolarisS3SignResponse.class)
34+
@JsonSerialize(as = ImmutablePolarisS3SignResponse.class)
35+
@SuppressWarnings("immutables:subtype")
36+
public interface PolarisS3SignResponse extends S3SignResponse {}

‎bom/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ dependencies {
2828
api(project(":polaris-api-iceberg-service"))
2929
api(project(":polaris-api-management-model"))
3030
api(project(":polaris-api-management-service"))
31+
api(project(":polaris-api-s3-sign-service"))
3132

3233
api(project(":polaris-container-spec-helper"))
3334
api(project(":polaris-minio-testcontainer"))

‎build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ tasks.register<Exec>("regeneratePythonClient") {
152152
dependsOn(":polaris-api-management-service:processResources")
153153
dependsOn(":polaris-api-catalog-service:processResources")
154154
dependsOn(":polaris-api-management-model:processResources")
155+
dependsOn(":polaris-api-s3-sign-service:processResources")
155156
}
156157

157158
// Pass environment variables:

‎gradle/projects.main.properties

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ polaris-api-iceberg-service=api/iceberg-service
2424
polaris-api-management-model=api/management-model
2525
polaris-api-management-service=api/management-service
2626
polaris-api-catalog-service=api/polaris-catalog-service
27+
polaris-api-s3-sign-service=api/s3-sign-service
2728
polaris-runtime-defaults=runtime/defaults
2829
polaris-runtime-service=runtime/service
2930
polaris-server=runtime/server

0 commit comments

Comments
 (0)