Reconciling azure terraform state with what is actually deployed to azure

infra/azure/terraform/capz/capz-monitoring/main.tf

@@ -26,13 +26,22 @@ variable "subscription_id" {
   type = string
 }
 
+locals {
+  # reproduce the previous pattern seen in state:
+  # "<first-10-of-rg>-<rg>-<first-6-of-subscription>"
+  computed_dns_prefix = format("%s-%s-%s",
+    substr(var.resource_group_name, 0, 10),
+    var.resource_group_name,
+    substr(var.subscription_id, 0, 6)
+  )
+}
+
 # Create the "capz-monitoring" resource group
 resource "azurerm_resource_group" "capz-monitoring" {
   location = var.location
   name     = var.resource_group_name
   tags = {
     DO-NOT-DELETE     = "contact capz"
-    creationTimestamp = timestamp()
   }
 }
 
@@ -49,8 +58,24 @@ resource "azurerm_role_assignment" "monitoring_reader" {
   depends_on = [ azurerm_user_assigned_identity.capz_monitoring_user_identity ]
 }
 
+# lookups for AKS-created user assigned identities and DNS zone (do not create new identities)
+data "azurerm_user_assigned_identity" "aks_akv" {
+  name                = "azurekeyvaultsecretsprovider-capz-monitoring"
+  resource_group_name = "MC_capz-monitoring_capz-monitoring_eastus"
+}
+
+data "azurerm_user_assigned_identity" "aks_webapp" {
+  name                = "webapprouting-capz-monitoring"
+  resource_group_name = "MC_capz-monitoring_capz-monitoring_eastus"
+}
+
+data "azurerm_dns_zone" "capz_monitoring" {
+  name                = "capz-monitoring.org"
+  resource_group_name = "capz-monitoring"
+}
+
 resource "azurerm_kubernetes_cluster" "capz-monitoring" {
-  dns_prefix            = var.resource_group_name
+  dns_prefix            = local.computed_dns_prefix
   location              = var.location
   name                  = var.resource_group_name
   resource_group_name   = var.resource_group_name
@@ -64,16 +89,48 @@ resource "azurerm_kubernetes_cluster" "capz-monitoring" {
   ]
   kubelet_identity {
     user_assigned_identity_id = azurerm_user_assigned_identity.capz_monitoring_user_identity.id
+    client_id                 = azurerm_user_assigned_identity.capz_monitoring_user_identity.client_id
+    object_id                 = azurerm_user_assigned_identity.capz_monitoring_user_identity.principal_id
   }
   identity {
     type                     = "UserAssigned"
     identity_ids = [
       azurerm_user_assigned_identity.capz_monitoring_user_identity.id
     ]
   }
+
+  # keep AKS addon-managed identities and the DNS zone referenced via data sources
+  key_vault_secrets_provider {
+    secret_rotation_enabled  = false
+    secret_rotation_interval = "2m"
+
+    # secret_identity is computed by the AKS provider; do not set it here.
+  }
+
+  web_app_routing {
+    default_nginx_controller = "AnnotationControlled"
+    dns_zone_ids = [
+      data.azurerm_dns_zone.capz_monitoring.id,
+    ]
+
+    # web_app_routing_identity is created/linked by AKS and is computed; do not set it here.
+  }
+
   default_node_pool {
-    name       = "default"
-    node_count = 1
-    vm_size    = "Standard_Ds2_v2"
+    name       = "nodepool1"
+    node_count = 3
+    vm_size    = "Standard_DS2_v2"
+
+    upgrade_settings {
+      drain_timeout_in_minutes      = 0
+      max_surge                     = "10%"
+      node_soak_duration_in_minutes = 0
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      linux_profile
+    ]
   }
 }

infra/azure/terraform/capz/cluster-api-gallery/main.tf

@@ -28,7 +28,7 @@ resource "azurerm_resource_group" "cluster-api-gallery" {
   name     = var.resource_group_name
   tags = {
     DO-NOT-DELETE     = "UpstreamInfra"
-    creationTimestamp = "2024-10-24T00:00:00Z"
+    creationTimestamp = "2024-10-03T15:53:21Z"
   }
 }
 
@@ -39,8 +39,9 @@ resource "azurerm_shared_image_gallery" "community_gallery" {
   name                = "community_gallery"
   resource_group_name = "cluster-api-gallery"
   tags = {
-    creationTimestamp = "2024-10-24T00:00:00Z"
+    creationTimestamp = "2024-10-24T17:36:37Z"
     jobName           = "image-builder-sig-ubuntu-2404"
+    DO-NOT-DELETE     = "UpstreamInfra"
   }
   sharing {
     permission = "Community"
@@ -62,6 +63,7 @@ resource "azurerm_user_assigned_identity" "pipelines_user_identity" {
   name                = "ado-pipeline-mi"
   resource_group_name = var.resource_group_name
   tags = {
+    DO-NOT-DELETE     = "UpstreamInfra"
     creationTimestamp = "2024-10-24T00:00:00Z"
   }
   depends_on = [

infra/azure/terraform/capz/container-registry/main.tf

@@ -29,10 +29,7 @@ resource "azurerm_container_registry" "capzci_registry" {
   sku                    = "Premium"
   anonymous_pull_enabled = true
 
-  retention_policy {
-    days    = 7
-    enabled = true
-  }
+  retention_policy_in_days = 7
 
   tags = {
     RetentionPolicy = "7days"
@@ -87,10 +84,7 @@ resource "azurerm_container_registry" "e2eprivate_registry" {
   resource_group_name = var.resource_group_name
   sku                 = "Premium"
 
-  retention_policy {
-    days    = 7
-    enabled = true
-  }
+  retention_policy_in_days = 7
 
   tags = {
     RetentionPolicy = "7days"

infra/azure/terraform/capz/identities/main.tf

@@ -22,6 +22,18 @@ variable "location" {
   type = string
 }
 
+variable "subscription_id" {
+  type = string
+}
+
+variable "container_registry_scope" {
+  type = string
+}
+
+variable "e2eprivate_registry_scope" {
+  type    = string
+}
+
 resource "azurerm_user_assigned_identity" "cloud_provider_user_identity" {
   name                = "cloud-provider-user-identity"
   location            = var.location
@@ -40,6 +52,31 @@ resource "azurerm_user_assigned_identity" "gmsa_user_identity" {
   resource_group_name = var.resource_group_name
 }
 
+resource "azurerm_role_definition" "gmsa_custom_role" {
+  name        = "gMSA"
+  scope       = "/subscriptions/${var.subscription_id}"
+  description = "Required permissions for gmsa to read properties of subscriptions and managed identities"
+
+  permissions {
+    actions = [
+      "Microsoft.Resources/subscriptions/read",
+      "Microsoft.ManagedIdentity/userAssignedIdentities/read"
+    ]
+    not_actions = []
+  }
+
+  assignable_scopes = [
+    "/subscriptions/${var.subscription_id}"
+  ]
+}
+
+resource "azurerm_role_assignment" "gmsa_role_assignment" {
+  principal_id   = azurerm_user_assigned_identity.domain_vm_identity.principal_id
+  role_definition_name = azurerm_role_definition.gmsa_custom_role.name
+  scope          = "/subscriptions/${var.subscription_id}"
+  depends_on     = [azurerm_user_assigned_identity.domain_vm_identity]
+}
+
 output "cloud_provider_user_identity_id" {
   value = azurerm_user_assigned_identity.cloud_provider_user_identity.principal_id
 }

infra/azure/terraform/capz/main.tf

@@ -16,6 +16,7 @@ limitations under the License.
 
 provider "azurerm" {
   features {}
+  subscription_id = var.subscription_id
 }
 
 terraform {
@@ -36,10 +37,6 @@ resource "azurerm_resource_provider_registration" "provider-k8s-config" {
   }
 }
 
-resource "azurerm_resource_provider_registration" "provider-container-service" {
-  name = "Microsoft.ContainerService"
-}
-
 resource "azurerm_marketplace_agreement" "traefik-agreement" {
   publisher = "containous"
   offer     = "traefik-proxy"
@@ -62,7 +59,6 @@ resource "azurerm_marketplace_agreement" "kinvolk-stable2-agreement" {
 # Data source to get the current client configuration
 data "azurerm_client_config" "current" {}
 
-
 # Resource group for CAPZ CI resources
 resource "azurerm_resource_group" "capz_ci" {
   location = var.location
@@ -80,45 +76,58 @@ resource "azurerm_storage_account" "k8sprowstorage" {
   min_tls_version                  = "TLS1_0"
   account_replication_type         = "RAGRS"
   cross_tenant_replication_enabled = true
-  depends_on                       = [azurerm_resource_group.capz_ci]
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
 }
 
-# Import identities module
-module "identities" {
-  source              = "./identities"
+# Import container registry module
+module "container_registry" {
+  source              = "./container-registry"
   resource_group_name = var.resource_group_name
   location            = var.location
-  depends_on          = [azurerm_resource_group.capz_ci]
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
+}
+
+# Import identities module
+module "identities" {
+  source                   = "./identities"
+  resource_group_name      = var.resource_group_name
+  location                 = var.location
+  subscription_id          = data.azurerm_client_config.current.subscription_id
+  container_registry_scope = module.container_registry.container_registry_id 
+  e2eprivate_registry_scope = module.container_registry.e2eprivate_registry_id
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
 }
 
 # Import key vault module
 module "key_vault" {
-  source              = "./key-vault"
-  resource_group_name = var.resource_group_name
-  location            = var.location
-  tenant_id           = data.azurerm_client_config.current.tenant_id
+  source                            = "./key-vault"
+  resource_group_name               = var.resource_group_name
+  location                          = var.location
+  tenant_id                         = data.azurerm_client_config.current.tenant_id  
   identities = {
-    cloud_provider_user_identity_id = module.identities.cloud_provider_user_identity_id
     domain_vm_identity_id           = module.identities.domain_vm_identity_id
     gmsa_user_identity_id           = module.identities.gmsa_user_identity_id
   }
-  depends_on = [azurerm_resource_group.capz_ci]
-}
-
-# Import container registry module
-module "container_registry" {
-  source              = "./container-registry"
-  resource_group_name = var.resource_group_name
-  location            = var.location
-  depends_on          = [azurerm_resource_group.capz_ci]
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
 }
 
 # Import role assignments module
 module "role_assignments" {
   source                   = "./role-assignments"
   resource_group_name      = var.resource_group_name
   container_registry_scope = module.container_registry.container_registry_id
-  subscription_id          = data.azurerm_client_config.current.subscription_id
+  e2eprivate_registry_scope = module.container_registry.e2eprivate_registry_id
+  cloud_provider_user_identity_id = module.identities.cloud_provider_user_identity_id
+  subscription_id          = data.azurerm_client_config.current.subscription_id 
+  key_vault_id             = module.key_vault.key_vault_id
   depends_on = [
     azurerm_resource_group.capz_ci,
     azurerm_storage_account.k8sprowstorage,
@@ -129,15 +138,17 @@ module "role_assignments" {
 # Import Cluster API gallery module
 module "cluster_api_gallery" {
   source              = "./cluster-api-gallery"
-  resource_group_name = var.resource_group_name
-  location            = var.location
-  depends_on          = module.role_assignments
+  resource_group_name = "cluster-api-gallery"
+  location            = "northcentralus"
+  depends_on          = [
+    module.role_assignments
+  ]
 }
 
 # Import CAPZ monitoring module
 module "capz_monitoring" {
   source              = "./capz-monitoring"
-  resource_group_name = var.resource_group_name
+  resource_group_name = "capz-monitoring"
   location            = var.location
   subscription_id     = data.azurerm_client_config.current.subscription_id
 }