WIP - Reconciling azure terraform state with what is actually deployed to azure

infra/azure/terraform/capz/capz-monitoring/main.tf

@@ -26,13 +26,22 @@ variable "subscription_id" {
   type = string
 }
 
+locals {
+  # reproduce the previous pattern seen in state:
+  # "<first-10-of-rg>-<rg>-<first-6-of-subscription>"
+  computed_dns_prefix = format("%s-%s-%s",
+    substr(var.resource_group_name, 0, 10),
+    var.resource_group_name,
+    substr(var.subscription_id, 0, 6)
+  )
+}
+
 # Create the "capz-monitoring" resource group
 resource "azurerm_resource_group" "capz-monitoring" {
   location = var.location
   name     = var.resource_group_name
   tags = {
     DO-NOT-DELETE     = "contact capz"
-    creationTimestamp = timestamp()
   }
 }
 
@@ -50,7 +59,7 @@ resource "azurerm_role_assignment" "monitoring_reader" {
 }
 
 resource "azurerm_kubernetes_cluster" "capz-monitoring" {
-  dns_prefix            = var.resource_group_name
+  dns_prefix            = local.computed_dns_prefix
   location              = var.location
   name                  = var.resource_group_name
   resource_group_name   = var.resource_group_name
@@ -64,6 +73,8 @@ resource "azurerm_kubernetes_cluster" "capz-monitoring" {
   ]
   kubelet_identity {
     user_assigned_identity_id = azurerm_user_assigned_identity.capz_monitoring_user_identity.id
+    client_id                 = azurerm_user_assigned_identity.capz_monitoring_user_identity.client_id
+    object_id                 = azurerm_user_assigned_identity.capz_monitoring_user_identity.principal_id
   }
   identity {
     type                     = "UserAssigned"
@@ -72,8 +83,14 @@ resource "azurerm_kubernetes_cluster" "capz-monitoring" {
     ]
   }
   default_node_pool {
-    name       = "default"
+    name       = "nodepool1"
     node_count = 1
-    vm_size    = "Standard_Ds2_v2"
+    vm_size    = "Standard_DS2_v2"
+  }
+
+  lifecycle {
+    ignore_changes = [
+      "linux_profile",
+    ]
   }
 }

infra/azure/terraform/capz/cluster-api-gallery/main.tf

@@ -62,6 +62,7 @@ resource "azurerm_user_assigned_identity" "pipelines_user_identity" {
   name                = "ado-pipeline-mi"
   resource_group_name = var.resource_group_name
   tags = {
+    DO-NOT-DELETE     = "UpstreamInfra"
     creationTimestamp = "2024-10-24T00:00:00Z"
   }
   depends_on = [

infra/azure/terraform/capz/container-registry/main.tf

@@ -29,10 +29,7 @@ resource "azurerm_container_registry" "capzci_registry" {
   sku                    = "Premium"
   anonymous_pull_enabled = true
 
-  retention_policy {
-    days    = 7
-    enabled = true
-  }
+  retention_policy_in_days = 7
 
   tags = {
     RetentionPolicy = "7days"
@@ -87,10 +84,7 @@ resource "azurerm_container_registry" "e2eprivate_registry" {
   resource_group_name = var.resource_group_name
   sku                 = "Premium"
 
-  retention_policy {
-    days    = 7
-    enabled = true
-  }
+  retention_policy_in_days = 7
 
   tags = {
     RetentionPolicy = "7days"

infra/azure/terraform/capz/identities/main.tf

@@ -22,6 +22,14 @@ variable "location" {
   type = string
 }
 
+variable "subscription_id" {
+  type = string
+}
+
+variable "container_registry_scope" {
+  type = string
+}
+
 resource "azurerm_user_assigned_identity" "cloud_provider_user_identity" {
   name                = "cloud-provider-user-identity"
   location            = var.location
@@ -40,6 +48,43 @@ resource "azurerm_user_assigned_identity" "gmsa_user_identity" {
   resource_group_name = var.resource_group_name
 }
 
+resource "azurerm_role_definition" "gmsa_custom_role" {
+  name        = "gMSA"
+  scope       = "/subscriptions/${var.subscription_id}"
+  description = "Required permissions for gmsa to read properties of subscriptions and managed identities"
+
+  permissions {
+    actions = [
+      "Microsoft.Resources/subscriptions/read",
+      "Microsoft.ManagedIdentity/userAssignedIdentities/read"
+    ]
+    not_actions = []
+  }
+
+  assignable_scopes = [
+    "/subscriptions/${var.subscription_id}"
+  ]
+}
+
+resource "azurerm_role_assignment" "gmsa_role_assignment" {
+  principal_id   = azurerm_user_assigned_identity.domain_vm_identity.principal_id
+  role_definition_name = azurerm_role_definition.gmsa_custom_role.name
+  scope          = "/subscriptions/${var.subscription_id}"
+  depends_on     = [azurerm_user_assigned_identity.domain_vm_identity]
+}
+
+resource "azurerm_role_assignment" "cloud_provider_sub_contributor" {
+  principal_id         = azurerm_user_assigned_identity.cloud_provider_user_identity.principal_id
+  role_definition_name = "Contributor"
+  scope                = "/subscriptions/${var.subscription_id}"
+}
+
+resource "azurerm_role_assignment" "acr_pull" {
+  principal_id         = azurerm_user_assigned_identity.cloud_provider_user_identity.principal_id
+  role_definition_name = "AcrPull"
+  scope                = var.container_registry_scope
+}
+
 output "cloud_provider_user_identity_id" {
   value = azurerm_user_assigned_identity.cloud_provider_user_identity.principal_id
 }

infra/azure/terraform/capz/main.tf

@@ -16,6 +16,7 @@ limitations under the License.
 
 provider "azurerm" {
   features {}
+  subscription_id = var.subscription_id
 }
 
 terraform {
@@ -62,7 +63,6 @@ resource "azurerm_marketplace_agreement" "kinvolk-stable2-agreement" {
 # Data source to get the current client configuration
 data "azurerm_client_config" "current" {}
 
-
 # Resource group for CAPZ CI resources
 resource "azurerm_resource_group" "capz_ci" {
   location = var.location
@@ -80,45 +80,56 @@ resource "azurerm_storage_account" "k8sprowstorage" {
   min_tls_version                  = "TLS1_0"
   account_replication_type         = "RAGRS"
   cross_tenant_replication_enabled = true
-  depends_on                       = [azurerm_resource_group.capz_ci]
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
 }
 
-# Import identities module
-module "identities" {
-  source              = "./identities"
+# Import container registry module
+module "container_registry" {
+  source              = "./container-registry"
   resource_group_name = var.resource_group_name
   location            = var.location
-  depends_on          = [azurerm_resource_group.capz_ci]
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
+}
+
+# Import identities module
+module "identities" {
+  source                   = "./identities"
+  resource_group_name      = var.resource_group_name
+  location                 = var.location
+  subscription_id          = data.azurerm_client_config.current.subscription_id
+  container_registry_scope = module.container_registry.container_registry_id 
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
 }
 
 # Import key vault module
 module "key_vault" {
-  source              = "./key-vault"
-  resource_group_name = var.resource_group_name
-  location            = var.location
-  tenant_id           = data.azurerm_client_config.current.tenant_id
+  source                            = "./key-vault"
+  resource_group_name               = var.resource_group_name
+  location                          = var.location
+  tenant_id                         = data.azurerm_client_config.current.tenant_id  
   identities = {
-    cloud_provider_user_identity_id = module.identities.cloud_provider_user_identity_id
     domain_vm_identity_id           = module.identities.domain_vm_identity_id
     gmsa_user_identity_id           = module.identities.gmsa_user_identity_id
   }
-  depends_on = [azurerm_resource_group.capz_ci]
-}
-
-# Import container registry module
-module "container_registry" {
-  source              = "./container-registry"
-  resource_group_name = var.resource_group_name
-  location            = var.location
-  depends_on          = [azurerm_resource_group.capz_ci]
+  depends_on = [
+    azurerm_resource_group.capz_ci
+  ]
 }
 
 # Import role assignments module
 module "role_assignments" {
   source                   = "./role-assignments"
   resource_group_name      = var.resource_group_name
   container_registry_scope = module.container_registry.container_registry_id
-  subscription_id          = data.azurerm_client_config.current.subscription_id
+  #storage_account_scope    = azurerm_storage_account.k8sprowstorage.id
+  subscription_id          = data.azurerm_client_config.current.subscription_id 
+  key_vault_id             = module.key_vault.key_vault_id
   depends_on = [
     azurerm_resource_group.capz_ci,
     azurerm_storage_account.k8sprowstorage,
@@ -129,15 +140,17 @@ module "role_assignments" {
 # Import Cluster API gallery module
 module "cluster_api_gallery" {
   source              = "./cluster-api-gallery"
-  resource_group_name = var.resource_group_name
-  location            = var.location
-  depends_on          = module.role_assignments
+  resource_group_name = "cluster-api-gallery"
+  location            = "northcentralus"
+  depends_on          = [
+    module.role_assignments
+  ]
 }
 
 # Import CAPZ monitoring module
 module "capz_monitoring" {
   source              = "./capz-monitoring"
-  resource_group_name = var.resource_group_name
+  resource_group_name = "capz-monitoring"
   location            = var.location
   subscription_id     = data.azurerm_client_config.current.subscription_id
 }

infra/azure/terraform/capz/role-assignments/main.tf

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+# This module maintains all role assignments for our service principal - az-cli-prow
+
 variable "resource_group_name" {
   type = string
 }
@@ -26,24 +28,28 @@ variable "subscription_id" {
   type = string
 }
 
+variable "key_vault_id" {
+  type = string
+} 
+
 data "azuread_service_principal" "az_service_principal" {
   display_name = "az-cli-prow"
 }
 
 resource "azurerm_role_assignment" "rg_contributor" {
-  principal_id         = data.azuread_service_principal.az_service_principal.id
+  principal_id         = data.azuread_service_principal.az_service_principal.object_id
   role_definition_name = "Contributor"
   scope                = "/subscriptions/${var.subscription_id}"
 }
 
 resource "azurerm_role_assignment" "storage_blob_data_contributor" {
-  principal_id         = data.azuread_service_principal.az_service_principal.id
+  principal_id         = data.azuread_service_principal.az_service_principal.object_id
   role_definition_name = "Storage Blob Data Contributor"
   scope                = "/subscriptions/${var.subscription_id}"
 }
 
 resource "azurerm_role_assignment" "acr_pull" {
-  principal_id         = data.azuread_service_principal.az_service_principal.id
+  principal_id         = data.azuread_service_principal.az_service_principal.object_id
   role_definition_name = "AcrPull"
   scope                = var.container_registry_scope
 }
@@ -65,7 +71,19 @@ resource "azurerm_role_definition" "custom_role" {
 }
 
 resource "azurerm_role_assignment" "sp_custom_role_assignment" {
-  principal_id         = data.azuread_service_principal.az_service_principal.id
+  principal_id         = data.azuread_service_principal.az_service_principal.object_id
   role_definition_name = azurerm_role_definition.custom_role.name
   scope                = "/subscriptions/${var.subscription_id}"
 }
+
+resource "azurerm_key_vault_access_policy" "access_policy_gmsa_sp" {
+  key_vault_id = var.key_vault_id
+  tenant_id    = data.azuread_service_principal.az_service_principal.application_tenant_id
+  object_id    = data.azuread_service_principal.az_service_principal.object_id
+  secret_permissions = [
+    "Get",
+    "Delete",
+    "List",
+    "Purge"
+  ]
+}

infra/azure/terraform/capz/variables.tf

@@ -31,3 +31,9 @@ variable "storage_account_name" {
   default     = "k8sprowstoragecomm"
   description = "Name of the storage account."
 }
+
+variable "subscription_id" {
+  type        = string
+  default     = "46678f10-4bbb-447e-98e8-d2829589f2d8"
+  description = "Azure Subscription ID to use for the azurerm provider."
+}