Merge pull request #90 from unity-sds/ogc-processes-api-register

OGC API Processes - Deploy and Undeploy Processes

Author: drewm-swe

airflow/docker/custom_airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.9.0-python3.11
+FROM apache/airflow:2.9.1-python3.11
 
 RUN pip install cwltool==3.1.20240112164112
 RUN pip install boto3==1.34.89

airflow/helm/values.tmpl.yaml

@@ -114,9 +114,9 @@ scheduler:
     component: scheduler
 
 triggerer:
-  replicas: 3
   keda:
     enabled: true
+    minReplicaCount: 1
 
 postgresql:
   enabled: false
@@ -258,6 +258,10 @@ dags:
     # the name of an existing PVC to use
     existingClaim: ${dags_pvc_name}
 
+dagProcessor:
+  enabled: true
+  replicas: 3
+
 env:
   - name: "AIRFLOW_VAR_KUBERNETES_PIPELINE_NAMESPACE"
     value: "${kubernetes_namespace}"
@@ -292,3 +296,7 @@ extraEnv: |
     value: "${webserver_navbar_color}"
   - name: AIRFLOW__WEBSERVER__INSTANCE_NAME
     value: "Deployment: ${webserver_instance_name}, ${service_area} Version: ${service_area_version}"
+  - name: AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL
+    value: "10"
+  - name: AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL
+    value: "5"

pyproject.toml

@@ -37,7 +37,7 @@ test = [
     "pytest-bdd==7.1.1",
     "pytest-mock==3.12.0",
     "requests==2.31.0",
-    "apache-airflow==2.8.1",
+    "apache-airflow==2.9.1",
     "kubernetes==29.0.0",
     "boto3==1.34.46",
     "backoff==2.2.1",

terraform-unity/README.md

@@ -178,14 +178,15 @@ No resources.
 |------|-------------|------|---------|:--------:|
 | <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
 | <a name="input_counter"></a> [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | `""` | no |
+| <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br>    url                 = string<br>    ref                 = string<br>    dags_directory_path = string<br>  })</pre> | <pre>{<br>  "dags_directory_path": "airflow/dags",<br>  "ref": "2.0.2-beta.0",<br>  "url": "https://github.com/unity-sds/unity-sps.git"<br>}</pre> | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    }),<br>    ogc_processes_api = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.0.1"<br>  },<br>  "ogc_processes_api": {<br>    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",<br>    "tag": "2.0.1"<br>  }<br>}</pre> | no |
-| <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "airflow": {<br>    "chart": "airflow",<br>    "repository": "https://airflow.apache.org",<br>    "version": "1.13.1"<br>  },<br>  "keda": {<br>    "chart": "keda",<br>    "repository": "https://kedacore.github.io/charts",<br>    "version": "v2.14.0"<br>  }<br>}</pre> | no |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    }),<br>    ogc_processes_api = object({<br>      name = string<br>      tag  = string<br>    })<br>    git_sync = object({<br>      name = string<br>      tag  = string<br>    })<br>    redis = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.0.2-beta.0"<br>  },<br>  "git_sync": {<br>    "name": "registry.k8s.io/git-sync/git-sync",<br>    "tag": "v4.2.3"<br>  },<br>  "ogc_processes_api": {<br>    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",<br>    "tag": "2.0.2-beta.0"<br>  },<br>  "redis": {<br>    "name": "redis",<br>    "tag": "7.2.4"<br>  }<br>}</pre> | no |
+| <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "airflow": {<br>    "chart": "airflow",<br>    "repository": "https://airflow.apache.org",<br>    "version": "1.13.1"<br>  },<br>  "keda": {<br>    "chart": "keda",<br>    "repository": "https://kedacore.github.io/charts",<br>    "version": "v2.14.2"<br>  }<br>}</pre> | no |
 | <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br>    requirements : list(object({<br>      key : string<br>      operator : string<br>      values : list(string)<br>    }))<br>    limits : object({<br>      cpu : string<br>      memory : string<br>    })<br>    disruption : object({<br>      consolidationPolicy : string<br>      consolidateAfter : string<br>    })<br>  }))</pre> | <pre>{<br>  "airflow-celery-workers": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "80",<br>      "memory": "320Gi"<br>    },<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "9"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-core-components": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "40",<br>      "memory": "160Gi"<br>    },<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "100",<br>      "memory": "400Gi"<br>    },<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "8191"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "32769"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  }<br>}</pre> | no |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |
 | <a name="input_mcp_ami_owner_id"></a> [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no |
-| <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | `"2.0.1"` | no |
+| <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | `"2.0.2-beta.0"` | no |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes |
 

terraform-unity/main.tf

@@ -12,4 +12,5 @@ module "unity-sps-airflow" {
   helm_charts                = var.helm_charts
   mcp_ami_owner_id           = var.mcp_ami_owner_id
   karpenter_node_pools       = var.karpenter_node_pools
+  dag_catalog_repo           = var.dag_catalog_repo
 }

terraform-unity/modules/terraform-unity-sps-airflow/README.md

@@ -35,7 +35,7 @@ No modules.
 | [aws_cloudwatch_log_group.airflow_dag_trigger](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/cloudwatch_log_group) | resource |
 | [aws_db_instance.sps_db](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/db_instance) | resource |
 | [aws_db_subnet_group.sps_db](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/db_subnet_group) | resource |
-| [aws_efs_access_point.airflow_dags](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/efs_access_point) | resource |
+| [aws_efs_access_point.airflow_deployed_dags](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/efs_access_point) | resource |
 | [aws_efs_access_point.airflow_kpo](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/efs_access_point) | resource |
 | [aws_efs_file_system.airflow](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/efs_file_system) | resource |
 | [aws_efs_mount_target.airflow](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/efs_mount_target) | resource |
@@ -74,24 +74,24 @@ No modules.
 | [aws_ssm_parameter.ogc_processes_ui_url](https://registry.terraform.io/providers/hashicorp/aws/5.47.0/docs/resources/ssm_parameter) | resource |
 | [helm_release.airflow](https://registry.terraform.io/providers/hashicorp/helm/2.13.1/docs/resources/release) | resource |
 | [helm_release.keda](https://registry.terraform.io/providers/hashicorp/helm/2.13.1/docs/resources/release) | resource |
-| [kubernetes_config_map.airflow_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/config_map) | resource |
 | [kubernetes_deployment.ogc_processes_api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/deployment) | resource |
+| [kubernetes_deployment.redis](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/deployment) | resource |
 | [kubernetes_ingress_v1.airflow_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource |
 | [kubernetes_ingress_v1.ogc_processes_api_ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/ingress_v1) | resource |
-| [kubernetes_job.copy_airflow_dags_to_pvc](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/job) | resource |
 | [kubernetes_manifest.karpenter_node_class](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/manifest) | resource |
 | [kubernetes_manifest.karpenter_node_pools](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/manifest) | resource |
 | [kubernetes_namespace.airflow](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/namespace) | resource |
 | [kubernetes_namespace.keda](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/namespace) | resource |
-| [kubernetes_persistent_volume.airflow_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume) | resource |
+| [kubernetes_persistent_volume.airflow_deployed_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume) | resource |
 | [kubernetes_persistent_volume.airflow_kpo](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume) | resource |
-| [kubernetes_persistent_volume_claim.airflow_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume_claim) | resource |
+| [kubernetes_persistent_volume_claim.airflow_deployed_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume_claim) | resource |
 | [kubernetes_persistent_volume_claim.airflow_kpo](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/persistent_volume_claim) | resource |
 | [kubernetes_role.airflow_pod_creator](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/role) | resource |
 | [kubernetes_role_binding.airflow_pod_creator_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/role_binding) | resource |
 | [kubernetes_secret.airflow_metadata](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/secret) | resource |
 | [kubernetes_secret.airflow_webserver](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/secret) | resource |
 | [kubernetes_service.ogc_processes_api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/service) | resource |
+| [kubernetes_service.redis](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/service) | resource |
 | [kubernetes_storage_class.efs](https://registry.terraform.io/providers/hashicorp/kubernetes/2.29.0/docs/resources/storage_class) | resource |
 | [null_resource.build_lambda_packages](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource |
 | [null_resource.remove_keda_finalizers](https://registry.terraform.io/providers/hashicorp/null/3.2.2/docs/resources/resource) | resource |
@@ -118,8 +118,9 @@ No modules.
 |------|-------------|------|---------|:--------:|
 | <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
 | <a name="input_counter"></a> [counter](#input\_counter) | Identifier used to uniquely distinguish resources. This is used in the naming convention of the resource. If left empty, a random hexadecimal value will be generated and used instead. | `string` | n/a | yes |
+| <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br>    url                 = string<br>    ref                 = string<br>    dags_directory_path = string<br>  })</pre> | n/a | yes |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | n/a | yes |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    }),<br>    ogc_processes_api = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | n/a | yes |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the associated services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    }),<br>    ogc_processes_api = object({<br>      name = string<br>      tag  = string<br>    })<br>    git_sync = object({<br>      name = string<br>      tag  = string<br>    })<br>    redis = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | n/a | yes |
 | <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | n/a | yes |
 | <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br>    requirements : list(object({<br>      key : string<br>      operator : string<br>      values : list(string)<br>    }))<br>    limits : object({<br>      cpu : string<br>      memory : string<br>    })<br>    disruption : object({<br>      consolidationPolicy : string<br>      consolidateAfter : string<br>    })<br>  }))</pre> | n/a | yes |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |