Skip to content

Commit 031e179

Browse files
owlleg6owlleg6
authored
Merge pull request #2 from data-platform-hq/fix/upgrade_to_terraform_v1.3_and_update_readme
fix: upgrade to terraform v1.3 and update readme
2 parents d5d2fc5 + d1744d1 commit 031e179

File tree

2 files changed

+239
-47
lines changed

2 files changed

+239
-47
lines changed

‎README.md‎

Lines changed: 238 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,22 @@ The main idea behind this module is to deploy resources for Databricks Workspace
88

99
Here we provide some examples of how to provision it with a different options.
1010

11+
### Example for Azure Cloud:
12+
1113
### In example below, these features of given module would be covered:
12-
1. Workspace admins assignment, custom Workspace group creation, group assignments, group entitlements
13-
2. Clusters (i.e., for Unity Catalog and Shared Autoscaling)
14-
3. Workspace IP Access list creation
15-
4. ADLS Gen2 Mount
16-
5. Create Secret Scope and assign permissions to custom groups
17-
6. SQL Endpoint creation and configuration
18-
7. Create Cluster policy
19-
8. Create an Azure Key Vault-backed secret scope
20-
9. Connect to already existing Unity Catalog Metastore
14+
1. Clusters (i.e., for Unity Catalog and Shared Autoscaling)
15+
2. Workspace IP Access list creation
16+
3. ADLS Gen2 Mount
17+
4. Create Secret Scope and assign permissions to custom groups
18+
5. SQL Endpoint creation and configuration
19+
6. Create Cluster policy
20+
7. Create an Azure Key Vault-backed secret scope
2121

2222
```hcl
2323
# Prerequisite resources
2424
25+
variable "databricks_account_id" {}
26+
2527
# Databricks Workspace with Premium SKU
2628
data "azurerm_databricks_workspace" "example" {
2729
name = "example-workspace"
@@ -35,46 +37,73 @@ provider "databricks" {
3537
azure_workspace_resource_id = data.azurerm_databricks_workspace.example.id
3638
}
3739
40+
# Databricks Account-Level Provider configuration
41+
provider "databricks" {
42+
alias = "account"
43+
host = "https://accounts.azuredatabricks.net"
44+
account_id = var.databricks_account_id
45+
}
46+
3847
# Key Vault where Service Principal's secrets are stored. Used for mounting Storage Container
3948
data "azurerm_key_vault" "example" {
4049
name = "example-key-vault"
4150
resource_group_name = "example-rg"
4251
}
4352
53+
locals {
54+
databricks_iam_account_groups = [{
55+
group_name = "example-gn"
56+
permissions = ["ADMIN"]
57+
entitlements = [
58+
"allow_instance_pool_create",
59+
"allow_cluster_create",
60+
"databricks_sql_access"
61+
]
62+
}]
63+
}
64+
65+
# Assigns Databricks Account groups to Workspace. It is required to assign Unity Catalog Metastore before assigning Account groups to Workspace
66+
module "databricks_account_groups" {
67+
count = length(local.databricks_iam_account_groups) != 0 ? 1 : 0
68+
source = "data-platform-hq/databricks-account-groups/databricks"
69+
version = "1.0.1"
70+
71+
workspace_id = data.azurerm_databricks_workspace.example.id
72+
workspace_group_assignment = local.databricks_iam_account_groups
73+
74+
providers = {
75+
databricks = databricks.account
76+
}
77+
}
78+
4479
# Example usage of module for Runtime Premium resources.
45-
module "databricks_runtime_premium" {
46-
source = "data-platform-hq/databricks-runtime-premium/databricks"
80+
module "databricks_runtime_premium" {
81+
source = "data-platform-hq/runtime/databricks"
82+
version = "~>1.0"
4783
4884
project = "datahq"
4985
env = "example"
5086
location = "eastus"
5187
52-
# Parameters of Service principal used for ADLS mount
53-
# Imports App ID and Secret of Service Principal from target Key Vault
54-
key_vault_id = data.azurerm_key_vault.example.id
55-
sp_client_id_secret_name = "sp-client-id" # secret's name that stores Service Principal App ID
56-
sp_key_secret_name = "sp-key" # secret's name that stores Service Principal Secret Key
57-
tenant_id_secret_name = "infra-arm-tenant-id" # secret's name that stores tenant id value
58-
59-
# 1.1 Workspace admins
60-
workspace_admins = {
61-
user = ["[email protected]"]
62-
service_principal = ["example-app-id"]
88+
# Cloud provider
89+
cloud_name = "azure"
90+
91+
# Example configuration for Workspace Groups
92+
iam_workspace_groups = {
93+
dev = {
94+
user = [
95+
"[email protected]",
96+
"[email protected]"
97+
]
98+
service_principal = []
99+
entitlements = ["allow_instance_pool_create","allow_cluster_create","databricks_sql_access"]
100+
}
63101
}
64102
65-
# 1.2 Custom Workspace group with assignments.
66-
# In addition, provides an ability to create group and entitlements.
67-
iam = [{
68-
group_name = "DEVELOPERS"
69-
permissions = ["ADMIN"]
70-
entitlements = [
71-
"allow_instance_pool_create",
72-
"allow_cluster_create",
73-
"databricks_sql_access"
74-
]
75-
}]
103+
# Example configuration for Account Groups
104+
iam_account_groups = local.databricks_iam_account_groups
76105
77-
# 2. Databricks clusters configuration, and assign permission to a custom group on clusters.
106+
# 1. Databricks clusters configuration, and assign permission to a custom group on clusters.
78107
databricks_cluster_configs = [ {
79108
cluster_name = "Unity Catalog"
80109
data_security_mode = "USER_ISOLATION"
@@ -90,33 +119,39 @@ module "databricks_runtime_premium" {
90119
permissions = [{group_name = "DEVELOPERS", permission_level = "CAN_MANAGE"}]
91120
}]
92121
93-
# 3. Workspace could be accessed only from these IP Addresses:
122+
# 2. Workspace could be accessed only from these IP Addresses:
94123
ip_rules = {
95124
"ip_range_1" = "10.128.0.0/16",
96125
"ip_range_2" = "10.33.0.0/16",
97126
}
98127
99-
# 4. ADLS Gen2 Mount
128+
# 3. ADLS Gen2 Mount
100129
mountpoints = {
101130
storage_account_name = data.azurerm_storage_account.example.name
102131
container_name = "example_container"
103132
}
104133
105-
# 5. Create Secret Scope and assign permissions to custom groups
134+
# Parameters of Service principal used for ADLS mount
135+
# Imports App ID and Secret of Service Principal from target Key Vault
136+
sp_client_id_secret_name = "sp-client-id" # secret's name that stores Service Principal App ID
137+
sp_key_secret_name = "sp-key" # secret's name that stores Service Principal Secret Key
138+
tenant_id_secret_name = "infra-arm-tenant-id" # secret's name that stores tenant id value
139+
140+
# 4. Create Secret Scope and assign permissions to custom groups
106141
secret_scope = [{
107142
scope_name = "extra-scope"
108143
acl = [{ principal = "DEVELOPERS", permission = "READ" }] # Only custom workspace group names are allowed. If left empty then only Workspace admins could access these keys
109144
secrets = [{ key = "secret-name", string_value = "secret-value"}]
110145
}]
111146
112-
# 6. SQL Warehouse Endpoint
147+
# 5. SQL Warehouse Endpoint
113148
databricks_sql_endpoint = [{
114149
name = "default"
115150
enable_serverless_compute = true
116151
permissions = [{ group_name = "DEVELOPERS", permission_level = "CAN_USE" },]
117152
}]
118153
119-
# 7. Databricks cluster policies
154+
# 6. Databricks cluster policies
120155
custom_cluster_policies = [{
121156
name = "custom_policy_1",
122157
can_use = "DEVELOPERS", # custom workspace group name, that is allowed to use this policy
@@ -129,7 +164,7 @@ module "databricks_runtime_premium" {
129164
}
130165
}]
131166
132-
# 8. Azure Key Vault-backed secret scope
167+
# 7. Azure Key Vault-backed secret scope
133168
key_vault_secret_scope = [{
134169
name = "external"
135170
key_vault_id = data.azurerm_key_vault.example.id
@@ -141,17 +176,174 @@ module "databricks_runtime_premium" {
141176
}
142177
}
143178
144-
# 9 Assignment already existing Unity Catalog Metastore
145-
module "metastore_assignment" {
146-
source = "data-platform-hq/metastore-assignment/databricks"
179+
```
180+
181+
### Example for AWS Cloud:
182+
183+
### In example below, these features of given module would be covered:
184+
1. Clusters (i.e., for Unity Catalog and Shared Autoscaling)
185+
2. Workspace IP Access list creation
186+
3. Create Secret Scope and assign permissions to custom groups
187+
4. SQL Endpoint creation and configuration
188+
5. Create Cluster policy
189+
190+
```hcl
191+
192+
# Prerequisite resources
193+
194+
variable "databricks_account_id" {}
195+
variable "region" {}
196+
197+
# Databricks Workspace ID
198+
data "databricks_mws_workspaces" "example" {
199+
account_id = var.databricks_account_id
200+
}
201+
202+
# Provider configuration for SSM
203+
provider "aws" {
204+
alias = "ssm"
205+
region = var.region
206+
}
207+
208+
# Databricks Account-Level Provider configuration
209+
provider "databricks" {
210+
alias = "mws"
211+
host = "https://accounts.cloud.databricks.com"
212+
account_id = data.aws_ssm_parameter.this["databricks_account_id"].value
213+
client_id = data.aws_ssm_parameter.this["databricks_admin_sp_id"].value
214+
client_secret = data.aws_ssm_parameter.this["databricks_admin_sp_secret"].value
215+
}
216+
217+
# Databricks Provider configuration
218+
provider "databricks" {
219+
alias = "workspace"
220+
host = module.databricks_workspace.workspace_url
221+
client_id = data.aws_ssm_parameter.this["databricks_admin_sp_id"].value
222+
client_secret = data.aws_ssm_parameter.this["databricks_admin_sp_secret"].value
223+
}
224+
225+
locals {
226+
ssm_parameters = [
227+
"databricks_account_id",
228+
"databricks_admin_sp_id",
229+
"databricks_admin_sp_secret",
230+
"github_pat_token"
231+
]
232+
233+
ssm_parameters_prefix = "/example-prefix/" # Prefix for parameters stored in AWS SSM
234+
235+
dbx_runtime = {
236+
iam_account_groups_assignment = [
237+
{ group_name = "example gm1", permissions = ["USER"] },
238+
{ group_name = "example gm2", permissions = ["USER"] }
239+
]
240+
241+
sql_endpoints = [{
242+
name = "example_test"
243+
permissions = [
244+
{ group_name = "example gm1", permission_level = "CAN_MANAGE" },
245+
]
246+
}]
247+
248+
clusters = [{
249+
cluster_name = "example1"
250+
permissions = [
251+
{ group_name = "example gm2", permission_level = "CAN_RESTART" },
252+
]
253+
}, {
254+
cluster_name = "example2"
255+
permissions = [
256+
{ group_name = "example gm2", permission_level = "CAN_RESTART" },
257+
{ group_name = "example gm1", permission_level = "CAN_MANAGE" },
258+
]
259+
}]
260+
}
261+
262+
databricks_custom_cluster_policies = [{
263+
name = null
264+
can_use = null
265+
definition = null
266+
}]
267+
268+
dbx_inputs = {
269+
vpc_id = "vpc-example"
270+
subnet_ids = ["subnet-example1", "subnet-example2"]
271+
security_group_ids = ["sg-example"]
272+
}
273+
274+
iam_default_permission_boundary_policy_arn = "arn:aws:iam::{ AWS Account ID }:policy/eo_role_boundary"
275+
}
276+
277+
# SSM Parameter
278+
data "aws_ssm_parameter" "this" {
279+
for_each = local.ssm_parameters
280+
name = "${local.ssm_parameters_prefix}${each.key}"
281+
provider = aws.ssm
282+
}
283+
284+
# Label configuration
285+
module "label" {
286+
source = "cloudposse/label/null"
287+
version = "0.25.0"
288+
289+
namespace = "example-namespace"
290+
environment = "example-environment"
291+
stage = "example-stage"
292+
}
293+
294+
# Databricks Workspace configuration
295+
module "databricks_workspace" {
296+
source = "data-platform-hq/aws-workspace/databricks"
297+
version = "1.0.1"
298+
299+
label = module.label.id
300+
vpc_id = local.dbx_inputs.vpc_id
301+
subnet_ids = local.dbx_inputs.subnet_ids
302+
security_group_ids = local.dbx_inputs.security_group_ids
303+
region = var.region
304+
account_id = data.aws_ssm_parameter.this["databricks_account_id"].value
305+
iam_cross_account_workspace_role_config = {
306+
permission_boundary_arn = local.iam_default_permission_boundary_policy_arn
307+
}
308+
309+
providers = {
310+
databricks = databricks.mws
311+
}
312+
}
313+
314+
# Account level group assignment to the Workspace
315+
module "databricks_account_groups" {
316+
source = "data-platform-hq/databricks-account-groups/databricks"
317+
version = "1.0.1"
318+
319+
workspace_id = module.databricks_workspace.workspace_id
320+
workspace_group_assignment = local.dbx_runtime.iam_account_groups_assignment
321+
322+
providers = {
323+
databricks = databricks.mws
324+
}
325+
}
326+
327+
# Databricks Runtime resources configuration (clusters, sql, secrets, etc.)
328+
module "databricks_runtime" {
329+
source = "data-platform-hq/runtime/databricks"
147330
version = "1.0.0"
148331
149-
workspace_id = data.azurerm_databricks_workspace.example.workspace_id
150-
metastore_id = "<uuid-of-metastore>"
332+
clusters = local.dbx_runtime.clusters
333+
sql_endpoint = local.dbx_runtime.sql_endpoints
334+
secret_scope = flatten([var.dbx_runtime.secret_scopes, local.demo_wwi_secret_scope])
335+
workspace_admin_token_enabled = var.workspace_admin_token_enabled
336+
system_schemas_enabled = alltrue([var.databricks_system_schemas_enabled])
337+
338+
iam_account_groups = local.dbx_runtime.iam_account_groups_assignment
339+
cloud_name = "aws"
340+
custom_cluster_policies = local.databricks_custom_cluster_policies
151341
152342
providers = {
153343
databricks = databricks.workspace
154344
}
345+
346+
depends_on = [module.databricks_workspace, module.databricks_account_groups]
155347
}
156348
157349
```
@@ -161,7 +353,7 @@ module "metastore_assignment" {
161353

162354
| Name | Version |
163355
|------|---------|
164-
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
356+
| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.3 |
165357
| <a name="requirement_databricks"></a> [databricks](#requirement\_databricks) | ~>1.0 |
166358

167359
## Providers

‎versions.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
terraform {
2-
required_version = ">= 1.0"
2+
required_version = ">=1.3"
33

44
required_providers {
55
databricks = {

0 commit comments

Comments
 (0)