Skip to content

Commit 0d6a451

Browse files
dimas-bdimas-b
committed
Rework with docker compose
1 parent f57ecdb commit 0d6a451

File tree

5 files changed

+296
-54
lines changed

5 files changed

+296
-54
lines changed

‎getting-started/assets/polaris/create-catalog.sh

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,14 @@ else
5757
echo "Using StorageType: $STORAGE_TYPE"
5858
fi
5959

60-
STORAGE_CONFIG_INFO="{\"storageType\": \"$STORAGE_TYPE\", \"allowedLocations\": [\"$STORAGE_LOCATION\"]}"
60+
if [ -z "${STORAGE_CONFIG_INFO}" ]; then
61+
STORAGE_CONFIG_INFO="{\"storageType\": \"$STORAGE_TYPE\", \"allowedLocations\": [\"$STORAGE_LOCATION\"]}"
6162

62-
if [[ "$STORAGE_TYPE" == "S3" ]]; then
63-
STORAGE_CONFIG_INFO=$(echo "$STORAGE_CONFIG_INFO" | jq --arg roleArn "$AWS_ROLE_ARN" '. + {roleArn: $roleArn}')
64-
elif [[ "$STORAGE_TYPE" == "AZURE" ]]; then
65-
STORAGE_CONFIG_INFO=$(echo "$STORAGE_CONFIG_INFO" | jq --arg tenantId "$AZURE_TENANT_ID" '. + {tenantId: $tenantId}')
63+
if [[ "$STORAGE_TYPE" == "S3" ]]; then
64+
STORAGE_CONFIG_INFO=$(echo "$STORAGE_CONFIG_INFO" | jq --arg roleArn "$AWS_ROLE_ARN" '. + {roleArn: $roleArn}')
65+
elif [[ "$STORAGE_TYPE" == "AZURE" ]]; then
66+
STORAGE_CONFIG_INFO=$(echo "$STORAGE_CONFIG_INFO" | jq --arg tenantId "$AZURE_TENANT_ID" '. + {tenantId: $tenantId}')
67+
fi
6668
fi
6769

6870
echo

‎getting-started/assets/polaris/obtain-token.sh

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
set -e
21+
22+
apk add --no-cache jq
23+
24+
realm=${1:-"POLARIS"}
25+
26+
TOKEN=$(curl -s http://polaris:8181/api/catalog/v1/oauth/tokens \
27+
--user ${CLIENT_ID}:${CLIENT_SECRET} \
28+
-H "Polaris-Realm: $realm" \
29+
-d grant_type=client_credentials \
30+
-d scope=PRINCIPAL_ROLE:ALL | jq -r .access_token)
31+
32+
if [ -z "${TOKEN}" ]; then
33+
echo "Failed to obtain access token."
34+
exit 1
35+
fi
36+
37+
export TOKEN

‎getting-started/minio/README.md

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
20+
# Getting Started with Apache Polaris and MinIO
21+
22+
## Overview
23+
24+
This example uses MinIO as a storage provider with Polaris.
25+
26+
Spark is used as a query engine. This example assumes a local Spark installation.
27+
See the [Spark Notebooks Example](../spark/README.md) for a more advanced Spark setup.
28+
29+
## Starting the Example
30+
31+
1. Build the Polaris server image if it's not already present locally:
32+
33+
```shell
34+
./gradlew \
35+
:polaris-server:assemble \
36+
:polaris-server:quarkusAppPartsBuild --rerun \
37+
-Dquarkus.container-image.build=true
38+
```
39+
40+
2. Start the docker compose group by running the following command from the root of the repository:
41+
42+
```shell
43+
docker compose -f getting-started/minio/docker-compose.yml up
44+
```
45+
46+
## Connecting From Spark
47+
48+
```shell
49+
bin/spark-sql \
50+
--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.iceberg:iceberg-aws-bundle:1.9.0 \
51+
--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
52+
--conf spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog \
53+
--conf spark.sql.catalog.polaris.type=rest \
54+
--conf spark.sql.catalog.polaris.uri=http://localhost:8181/api/catalog \
55+
--conf spark.sql.catalog.polaris.token-refresh-enabled=false \
56+
--conf spark.sql.catalog.polaris.warehouse=quickstart_catalog \
57+
--conf spark.sql.catalog.polaris.scope=PRINCIPAL_ROLE:ALL \
58+
--conf spark.sql.catalog.polaris.header.X-Iceberg-Access-Delegation=vended-credentials \
59+
--conf spark.sql.catalog.polaris.credential=root:s3cr3t
60+
```
61+
62+
Note: `s3cr3t` is defined as the password for the `root` users in the `docker-compose.yml` file.
63+
64+
## Running Queries
65+
66+
Run inside the Spark SQL shell:
67+
68+
```
69+
spark-sql (default)> use polaris;
70+
Time taken: 0.837 seconds
71+
72+
spark-sql ()> create namespace ns;
73+
Time taken: 0.374 seconds
74+
75+
spark-sql ()> create table ns.t1 as select 'abc';
76+
Time taken: 2.192 seconds
77+
78+
spark-sql ()> select * from ns.t1;
79+
abc
80+
Time taken: 0.579 seconds, Fetched 1 row(s)
81+
```
82+
83+
## MinIO Endpoints
84+
85+
Note that the catalog configuration defined in the `docker-compose.yml` contains
86+
different endpoints for the Polaris Server and the client (Spark). Specifically,
87+
the client endpoint is `http://localhost:9000`, but `endpointInternal` is `http://minio:9000`.
88+
89+
This is necessary because clients running on `localhost` do not normally see service
90+
names (such as `minio`) that are internal to the docker compose environment.

‎getting-started/minio/docker-compose.yml

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
services:
21+
22+
minio:
23+
image: quay.io/minio/minio:latest
24+
ports:
25+
# API port
26+
- "9000:9000"
27+
# UI port
28+
- "9001:9001"
29+
environment:
30+
MINIO_ROOT_USER: minio_root
31+
MINIO_ROOT_PASSWORD: m1n1opwd
32+
command:
33+
- "server"
34+
- "/data"
35+
- "--console-address"
36+
- ":9001"
37+
healthcheck:
38+
test: ["CMD", "curl", "http://127.0.0.1:9000/minio/health/live"]
39+
interval: 1s
40+
timeout: 10s
41+
42+
polaris:
43+
image: apache/polaris:latest
44+
ports:
45+
# API port
46+
- "8181:8181"
47+
# Optional, allows attaching a debugger to the Polaris JVM
48+
- "5005:5005"
49+
depends_on:
50+
minio:
51+
condition: service_healthy
52+
environment:
53+
JAVA_DEBUG: true
54+
JAVA_DEBUG_PORT: "*:5005"
55+
AWS_REGION: us-west-2
56+
AWS_ACCESS_KEY_ID: minio_root
57+
AWS_SECRET_ACCESS_KEY: m1n1opwd
58+
POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,s3cr3t
59+
polaris.realm-context.realms: POLARIS
60+
quarkus.otel.sdk.disabled: "true"
61+
healthcheck:
62+
test: ["CMD", "curl", "http://localhost:8182/q/health"]
63+
interval: 2s
64+
timeout: 10s
65+
retries: 10
66+
start_period: 10s
67+
68+
setup_bucket:
69+
image: quay.io/minio/mc:latest
70+
depends_on:
71+
minio:
72+
condition: service_healthy
73+
entrypoint: "/bin/sh"
74+
command:
75+
- "-c"
76+
- >-
77+
echo Creating MinIO bucket...;
78+
mc alias set pol http://minio:9000 minio_root m1n1opwd;
79+
mc mb pol/bucket123;
80+
mc ls pol;
81+
echo Bucket setup complete.;
82+
83+
polaris-setup:
84+
image: alpine/curl
85+
depends_on:
86+
polaris:
87+
condition: service_healthy
88+
environment:
89+
- CLIENT_ID=root
90+
- CLIENT_SECRET=s3cr3t
91+
volumes:
92+
- ../assets/polaris/:/polaris
93+
entrypoint: "/bin/sh"
94+
command:
95+
- "-c"
96+
- >-
97+
chmod +x /polaris/create-catalog.sh;
98+
chmod +x /polaris/obtain-token.sh;
99+
source /polaris/obtain-token.sh;
100+
echo Creating catalog...;
101+
export STORAGE_CONFIG_INFO='{"storageType":"S3",
102+
"endpoint":"http://localhost:9000",
103+
"endpointInternal":"http://minio:9000",
104+
"pathStyleAccess":true}';
105+
export STORAGE_LOCATION='s3://bucket123';
106+
/polaris/create-catalog.sh POLARIS $$TOKEN;
107+
echo Extra grants...;
108+
curl -H "Authorization: Bearer $$TOKEN" -H 'Content-Type: application/json' \
109+
-X PUT \
110+
http://polaris:8181/api/management/v1/catalogs/quickstart_catalog/catalog-roles/catalog_admin/grants \
111+
-d '{"type":"catalog", "privilege":"CATALOG_MANAGE_CONTENT"}';
112+
echo Done.;

0 commit comments

Comments
 (0)