Skip to content

Commit 266c8d5

Browse files
author
Drew Meyers
committed
Make (un)deploy more fault tolerant
1 parent ab0dcaa commit 266c8d5

File tree

5 files changed

+81
-9
lines changed

5 files changed

+81
-9
lines changed

‎terraform-unity/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ No resources.
180180
| <a name="input_counter"></a> [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | `""` | no |
181181
| <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br> url = string<br> ref = string<br> dags_directory_path = string<br> })</pre> | <pre>{<br> "dags_directory_path": "airflow/dags",<br> "ref": "2.0.2-beta.0",<br> "url": "https://github.com/unity-sds/unity-sps.git"<br>}</pre> | no |
182182
| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
183-
| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br> airflow = object({<br> name = string<br> tag = string<br> }),<br> ogc_processes_api = object({<br> name = string<br> tag = string<br> })<br> git_sync = object({<br> name = string<br> tag = string<br> })<br> })</pre> | <pre>{<br> "airflow": {<br> "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br> "tag": "2.0.2-beta.0"<br> },<br> "git_sync": {<br> "name": "registry.k8s.io/git-sync/git-sync",<br> "tag": "v4.2.3"<br> },<br> "ogc_processes_api": {<br> "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",<br> "tag": "2.0.2-beta.0"<br> }<br>}</pre> | no |
183+
| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br> airflow = object({<br> name = string<br> tag = string<br> }),<br> ogc_processes_api = object({<br> name = string<br> tag = string<br> })<br> git_sync = object({<br> name = string<br> tag = string<br> })<br> redis = object({<br> name = string<br> tag = string<br> })<br> })</pre> | <pre>{<br> "airflow": {<br> "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br> "tag": "2.0.2-beta.0"<br> },<br> "git_sync": {<br> "name": "registry.k8s.io/git-sync/git-sync",<br> "tag": "v4.2.3"<br> },<br> "ogc_processes_api": {<br> "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",<br> "tag": "2.0.2-beta.0"<br> },<br> "redis": {<br> "name": "redis",<br> "tag": "7.2.4"<br> }<br>}</pre> | no |
184184
| <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br> repository = string<br> chart = string<br> version = string<br> }))</pre> | <pre>{<br> "airflow": {<br> "chart": "airflow",<br> "repository": "https://airflow.apache.org",<br> "version": "1.13.1"<br> },<br> "keda": {<br> "chart": "keda",<br> "repository": "https://kedacore.github.io/charts",<br> "version": "v2.14.2"<br> }<br>}</pre> | no |
185185
| <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br> requirements : list(object({<br> key : string<br> operator : string<br> values : list(string)<br> }))<br> limits : object({<br> cpu : string<br> memory : string<br> })<br> disruption : object({<br> consolidationPolicy : string<br> consolidateAfter : string<br> })<br> }))</pre> | <pre>{<br> "airflow-celery-workers": {<br> "disruption": {<br> "consolidateAfter": "1m",<br> "consolidationPolicy": "WhenEmpty"<br> },<br> "limits": {<br> "cpu": "80",<br> "memory": "320Gi"<br> },<br> "requirements": [<br> {<br> "key": "karpenter.k8s.aws/instance-family",<br> "operator": "In",<br> "values": [<br> "m7i",<br> "m6i",<br> "m5",<br> "t3",<br> "c7i",<br> "c6i",<br> "c5",<br> "r7i",<br> "r6i",<br> "r5"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-cpu",<br> "operator": "Gt",<br> "values": [<br> "1"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-cpu",<br> "operator": "Lt",<br> "values": [<br> "9"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-memory",<br> "operator": "Gt",<br> "values": [<br> "8191"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-memory",<br> "operator": "Lt",<br> "values": [<br> "32769"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-hypervisor",<br> "operator": "In",<br> "values": [<br> "nitro"<br> ]<br> }<br> ]<br> },<br> "airflow-core-components": {<br> "disruption": {<br> "consolidateAfter": "1m",<br> "consolidationPolicy": "WhenEmpty"<br> },<br> "limits": {<br> "cpu": "40",<br> "memory": "160Gi"<br> },<br> "requirements": [<br> {<br> "key": "karpenter.k8s.aws/instance-family",<br> "operator": "In",<br> "values": [<br> "m7i",<br> "m6i",<br> "m5",<br> "t3",<br> "c7i",<br> "c6i",<br> "c5",<br> "r7i",<br> "r6i",<br> "r5"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-cpu",<br> "operator": "Gt",<br> "values": [<br> "1"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-cpu",<br> "operator": "Lt",<br> "values": [<br> "17"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-memory",<br> "operator": "Gt",<br> "values": [<br> "8191"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-memory",<br> "operator": "Lt",<br> "values": [<br> "32769"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-hypervisor",<br> "operator": "In",<br> "values": [<br> "nitro"<br> ]<br> }<br> ]<br> },<br> "airflow-kubernetes-pod-operator": {<br> "disruption": {<br> "consolidateAfter": "1m",<br> "consolidationPolicy": "WhenEmpty"<br> },<br> "limits": {<br> "cpu": "100",<br> "memory": "400Gi"<br> },<br> "requirements": [<br> {<br> "key": "karpenter.k8s.aws/instance-family",<br> "operator": "In",<br> "values": [<br> "m7i",<br> "m6i",<br> "m5",<br> "t3",<br> "c7i",<br> "c6i",<br> "c5",<br> "r7i",<br> "r6i",<br> "r5"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-cpu",<br> "operator": "Gt",<br> "values": [<br> "1"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-cpu",<br> "operator": "Lt",<br> "values": [<br> "17"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-memory",<br> "operator": "Gt",<br> "values": [<br> "8191"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-memory",<br> "operator": "Lt",<br> "values": [<br> "32769"<br> ]<br> },<br> {<br> "key": "karpenter.k8s.aws/instance-hypervisor",<br> "operator": "In",<br> "values": [<br> "nitro"<br> ]<br> }<br> ]<br> }<br>}</pre> | no |
186186
| <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |

‎terraform-unity/modules/terraform-unity-sps-airflow/README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ No modules.
7575
| [helm_release.airflow](https://registry.terraform.io/providers/hashicorp/helm/2.13.1/docs/resources/release) | resource |
7676
| [helm_release.keda](https://registry.terraform.io/providers/hashicorp/helm/2.13.1/docs/resources/release) | resource |
7777
| [kubernetes_deployment.ogc_processes_api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/deployment) | resource |
78+
| [kubernetes_deployment.redis](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/deployment) | resource |
7879
| [kubernetes_ingress_v1.airflow_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource |
7980
| [kubernetes_ingress_v1.ogc_processes_api_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource |
8081
| [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/manifest) | resource |
@@ -90,6 +91,7 @@ No modules.
9091
| [kubernetes_secret.airflow_metadata](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/secret) | resource |
9192
| [kubernetes_secret.airflow_webserver](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/secret) | resource |
9293
| [kubernetes_service.ogc_processes_api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/service) | resource |
94+
| [kubernetes_service.redis](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/service) | resource |
9395
| [kubernetes_storage_class.efs](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/storage_class) | resource |
9496
| [null_resource.build_lambda_packages](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource |
9597
| [null_resource.remove_keda_finalizers](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource |
@@ -118,7 +120,7 @@ No modules.
118120
| <a name="input_counter"></a> [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | n/a | yes |
119121
| <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br> url = string<br> ref = string<br> dags_directory_path = string<br> })</pre> | n/a | yes |
120122
| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
121-
| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br> airflow = object({<br> name = string<br> tag = string<br> }),<br> ogc_processes_api = object({<br> name = string<br> tag = string<br> })<br> git_sync = object({<br> name = string<br> tag = string<br> })<br> })</pre> | n/a | yes |
123+
| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br> airflow = object({<br> name = string<br> tag = string<br> }),<br> ogc_processes_api = object({<br> name = string<br> tag = string<br> })<br> git_sync = object({<br> name = string<br> tag = string<br> })<br> redis = object({<br> name = string<br> tag = string<br> })<br> })</pre> | n/a | yes |
122124
| <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br> repository = string<br> chart = string<br> version = string<br> }))</pre> | n/a | yes |
123125
| <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br> requirements : list(object({<br> key : string<br> operator : string<br> values : list(string)<br> }))<br> limits : object({<br> cpu : string<br> memory : string<br> })<br> disruption : object({<br> consolidationPolicy : string<br> consolidateAfter : string<br> })<br> }))</pre> | n/a | yes |
124126
| <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |

‎terraform-unity/modules/terraform-unity-sps-airflow/main.tf

Lines changed: 65 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,55 @@ resource "helm_release" "airflow" {
473473
]
474474
}
475475

476+
resource "kubernetes_deployment" "redis" {
477+
metadata {
478+
name = "ogc-processes-api-redis-lock"
479+
namespace = kubernetes_namespace.airflow.metadata[0].name
480+
}
481+
spec {
482+
replicas = 1
483+
selector {
484+
match_labels = {
485+
app = "redis"
486+
}
487+
}
488+
template {
489+
metadata {
490+
labels = {
491+
app = "redis"
492+
}
493+
}
494+
spec {
495+
container {
496+
name = "redis"
497+
image = "${var.docker_images.redis.name}:${var.docker_images.redis.tag}"
498+
port {
499+
container_port = 6379
500+
}
501+
}
502+
}
503+
}
504+
}
505+
}
506+
507+
resource "kubernetes_service" "redis" {
508+
metadata {
509+
name = "ogc-processes-api-redis-lock"
510+
namespace = kubernetes_namespace.airflow.metadata[0].name
511+
}
512+
spec {
513+
selector = {
514+
app = "redis"
515+
}
516+
port {
517+
name = "redis"
518+
port = 6379
519+
target_port = 6379
520+
}
521+
type = "ClusterIP"
522+
}
523+
}
524+
476525
resource "kubernetes_deployment" "ogc_processes_api" {
477526
metadata {
478527
name = "ogc-processes-api"
@@ -521,33 +570,42 @@ resource "kubernetes_deployment" "ogc_processes_api" {
521570
}
522571
}
523572
container {
524-
image = "${var.docker_images.ogc_processes_api.name}:${var.docker_images.ogc_processes_api.tag}"
525573
name = "ogc-processes-api"
574+
image = "${var.docker_images.ogc_processes_api.name}:${var.docker_images.ogc_processes_api.tag}"
526575
port {
527576
container_port = 80
528577
}
529578
env {
530-
name = "db_url"
579+
name = "DB_URL"
531580
value = "postgresql://${aws_db_instance.sps_db.username}:${urlencode(aws_secretsmanager_secret_version.sps_db.secret_string)}@${aws_db_instance.sps_db.endpoint}/${aws_db_instance.sps_db.db_name}"
532581
}
533582
env {
534-
name = "ems_api_url"
583+
name = "REDIS_HOST"
584+
value = "${kubernetes_service.redis.metadata[0].name}.${kubernetes_namespace.airflow.metadata[0].name}.svc.cluster.local"
585+
586+
}
587+
env {
588+
name = "REDIS_PORT"
589+
value = "6379"
590+
}
591+
env {
592+
name = "EMS_API_URL"
535593
value = aws_ssm_parameter.airflow_api_url.value
536594
}
537595
env {
538-
name = "ems_api_auth_username"
596+
name = "EMS_API_AUTH_USERNAME"
539597
value = local.airflow_webserver_username
540598
}
541599
env {
542-
name = "ems_api_auth_password"
600+
name = "EMS_API_AUTH_PASSWORD"
543601
value = var.airflow_webserver_password
544602
}
545603
env {
546-
name = "dag_catalog_directory"
604+
name = "DAG_CATALOG_DIRECTORY"
547605
value = "/dag-catalog/current/${var.dag_catalog_repo.dags_directory_path}"
548606
}
549607
env {
550-
name = "deployed_dags_directory"
608+
name = "DEPLOYED_DAGS_DIRECTORY"
551609
value = "/deployed-dags"
552610
}
553611
volume_mount {

‎terraform-unity/modules/terraform-unity-sps-airflow/variables.tf

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ variable "docker_images" {
6262
name = string
6363
tag = string
6464
})
65+
redis = object({
66+
name = string
67+
tag = string
68+
})
6569
})
6670
}
6771

‎terraform-unity/variables.tf

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ variable "docker_images" {
8282
name = string
8383
tag = string
8484
})
85+
redis = object({
86+
name = string
87+
tag = string
88+
})
8589
})
8690
default = {
8791
airflow = {
@@ -95,6 +99,10 @@ variable "docker_images" {
9599
git_sync = {
96100
name = "registry.k8s.io/git-sync/git-sync"
97101
tag = "v4.2.3"
102+
},
103+
redis = {
104+
name = "redis"
105+
tag = "7.2.4"
98106
}
99107
}
100108
}

0 commit comments

Comments
 (0)