Upgrading the Dcker images to version 2.5.8

Author: LucaCinquini

airflow/plugins/unity_sps_utils.py

@@ -16,7 +16,7 @@
 
 # Note: each Pod is assigned the same label to assure that (via the anti-affinity requirements)
 # two Pods with the same label cannot run on the same Node
-SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.5.7"
+SPS_DOCKER_CWL_IMAGE = "ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.5.8"
 
 NODE_POOL_DEFAULT = "airflow-kubernetes-pod-operator"
 NODE_POOL_HIGH_WORKLOAD = "airflow-kubernetes-pod-operator-high-workload"

terraform-unity/README.md

@@ -188,15 +188,15 @@ terraform apply -no-color 2>&1 | tee apply_output.txt
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_airflow_docker_images"></a> [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.5.7"<br>  }<br>}</pre> | no |
+| <a name="input_airflow_docker_images"></a> [airflow\_docker\_images](#input\_airflow\_docker\_images) | Docker images for the associated Airflow services. | <pre>object({<br>    airflow = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "airflow": {<br>    "name": "ghcr.io/unity-sds/unity-sps/sps-airflow",<br>    "tag": "2.5.8"<br>  }<br>}</pre> | no |
 | <a name="input_airflow_webserver_password"></a> [airflow\_webserver\_password](#input\_airflow\_webserver\_password) | The password for the Airflow webserver and UI. | `string` | n/a | yes |
 | <a name="input_airflow_webserver_username"></a> [airflow\_webserver\_username](#input\_airflow\_webserver\_username) | The username for the Airflow webserver and UI. | `string` | `"admin"` | no |
 | <a name="input_dag_catalog_repo"></a> [dag\_catalog\_repo](#input\_dag\_catalog\_repo) | Git repository that stores the catalog of Airflow DAGs. | <pre>object({<br>    url                 = string<br>    ref                 = string<br>    dags_directory_path = string<br>  })</pre> | <pre>{<br>  "dags_directory_path": "airflow/dags",<br>  "ref": "develop",<br>  "url": "https://github.com/unity-sds/unity-sps.git"<br>}</pre> | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | `""` | no |
 | <a name="input_helm_charts"></a> [helm\_charts](#input\_helm\_charts) | Helm charts for the associated services. | <pre>map(object({<br>    repository = string<br>    chart      = string<br>    version    = string<br>  }))</pre> | <pre>{<br>  "airflow": {<br>    "chart": "airflow",<br>    "repository": "https://airflow.apache.org",<br>    "version": "1.15.0"<br>  },<br>  "keda": {<br>    "chart": "keda",<br>    "repository": "https://kedacore.github.io/charts",<br>    "version": "v2.15.1"<br>  }<br>}</pre> | no |
 | <a name="input_installprefix"></a> [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no |
 | <a name="input_karpenter_node_classes"></a> [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | Configuration for karpenter\_node\_classes | <pre>map(object({<br>    volume_size = string<br>  }))</pre> | <pre>{<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "volume_size": "300Gi"<br>  },<br>  "default": {<br>    "volume_size": "30Gi"<br>  }<br>}</pre> | no |
-| <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br>    requirements : list(object({<br>      key : string<br>      operator : string<br>      values : list(string)<br>    }))<br>    nodeClassRef : string<br>    limits : object({<br>      cpu : string<br>      memory : string<br>    })<br>    disruption : object({<br>      consolidationPolicy : string<br>      consolidateAfter : string<br>    })<br>  }))</pre> | <pre>{<br>  "airflow-celery-workers": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "80",<br>      "memory": "320Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5",<br>          "m5ad"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "4095"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "65537"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-core-components": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "40",<br>      "memory": "160Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5",<br>          "m5ad"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "4095"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "65537"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "6400",<br>      "memory": "12800Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "m5ad",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "0"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "511"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "65537"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "6400",<br>      "memory": "12800Gi"<br>    },<br>    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "m5ad",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "0"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "65"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "511"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "262145"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  }<br>}</pre> | no |
+| <a name="input_karpenter_node_pools"></a> [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | <pre>map(object({<br>    requirements : list(object({<br>      key : string<br>      operator : string<br>      values : list(string)<br>    }))<br>    nodeClassRef : string<br>    limits : object({<br>      cpu : string<br>      memory : string<br>    })<br>    disruption : object({<br>      consolidationPolicy : string<br>      consolidateAfter : string<br>    })<br>  }))</pre> | <pre>{<br>  "airflow-celery-workers": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "80",<br>      "memory": "320Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c6id",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5",<br>          "m5ad"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "4095"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "65537"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-core-components": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "40",<br>      "memory": "160Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c6id",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5",<br>          "m5ad"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "1"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "4095"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "65537"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "6400",<br>      "memory": "12800Gi"<br>    },<br>    "nodeClassRef": "default",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "m5ad",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c6id",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "0"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "17"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "511"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "65537"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  },<br>  "airflow-kubernetes-pod-operator-high-workload": {<br>    "disruption": {<br>      "consolidateAfter": "1m",<br>      "consolidationPolicy": "WhenEmpty"<br>    },<br>    "limits": {<br>      "cpu": "6400",<br>      "memory": "12800Gi"<br>    },<br>    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",<br>    "requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-family",<br>        "operator": "In",<br>        "values": [<br>          "m7i",<br>          "m6i",<br>          "m5",<br>          "m5ad",<br>          "t3",<br>          "c7i",<br>          "c6i",<br>          "c6id",<br>          "c5",<br>          "r7i",<br>          "r6i",<br>          "r5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Gt",<br>        "values": [<br>          "0"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "Lt",<br>        "values": [<br>          "65"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Gt",<br>        "values": [<br>          "511"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-memory",<br>        "operator": "Lt",<br>        "values": [<br>          "262145"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-hypervisor",<br>        "operator": "In",<br>        "values": [<br>          "nitro"<br>        ]<br>      }<br>    ]<br>  }<br>}</pre> | no |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes |
 | <a name="input_mcp_ami_owner_id"></a> [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no |
 | <a name="input_ogc_processes_docker_images"></a> [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | <pre>object({<br>    ogc_processes_api = object({<br>      name = string<br>      tag  = string<br>    })<br>    git_sync = object({<br>      name = string<br>      tag  = string<br>    })<br>    redis = object({<br>      name = string<br>      tag  = string<br>    })<br>  })</pre> | <pre>{<br>  "git_sync": {<br>    "name": "registry.k8s.io/git-sync/git-sync",<br>    "tag": "v4.2.4"<br>  },<br>  "ogc_processes_api": {<br>    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",<br>    "tag": "2.0.0"<br>  },<br>  "redis": {<br>    "name": "redis",<br>    "tag": "7.4.0"<br>  }<br>}</pre> | no |

terraform-unity/variables.tf

@@ -73,7 +73,7 @@ variable "airflow_docker_images" {
   default = {
     airflow = {
       name = "ghcr.io/unity-sds/unity-sps/sps-airflow"
-      tag  = "2.5.7"
+      tag  = "2.5.8"
     }
   }
 }