feat: Support existing postgres database in different existing resour…

examples/README.md

@@ -113,3 +113,210 @@ custom_storage_account_name = "production-${local.deployment_name}-storage-accou
 3. **Check Lengths**: Verify storage account and key vault names are ≤24 characters
 4. **Document**: Include comments explaining your naming convention
 5. **Verify**: Check Azure portal after deployment to ensure resources have expected names and permissions work
+
+## Using Existing PostgreSQL Database
+
+The module supports using an existing PostgreSQL database instead of creating a new one. This is useful when:
+
+- You have a shared database server across multiple environments
+- You want to use a centrally managed database instance
+- The database exists in a different resource group or subscription
+
+### Configuration
+
+To use an existing database, set the following variables:
+
+```hcl
+module "azure" {
+  # ... other configuration ...
+
+  # Enable database module but use existing database
+  create_database       = true
+  use_existing_database = true
+
+  # Specify the existing database details
+  existing_database_resource_group_name = "my-database-rg"
+  existing_postgresql_server_name       = "my-postgres-server"
+  existing_postgresql_database_name     = "datafold"
+}
+```
+
+### Important Notes
+
+1. **Authentication**: The module cannot access the password for existing databases. You'll need to manage authentication separately.
+
+2. **Resource Group**: The existing database can be in a different resource group that this module doesn't manage.
+
+3. **Networking**: Ensure the AKS cluster can reach the existing database (proper networking, firewall rules, etc.).
+
+4. **Validation**: All three existing database variables must be provided when `use_existing_database = true`.
+
+### Example
+
+```hcl
+locals {
+  # Use existing database in production, create new in dev
+  use_existing_db = var.environment == "production"
+}
+
+module "azure" {
+  source = "path/to/terraform-azure-datafold"
+
+  deployment_name = "my-app-${var.environment}"
+
+  # Database configuration
+  create_database       = true
+  use_existing_database = local.use_existing_db
+
+  # Existing database details (only used when use_existing_database = true)
+  existing_database_resource_group_name = local.use_existing_db ? "production-shared-db-rg" : ""
+  existing_postgresql_server_name       = local.use_existing_db ? "prod-postgres-shared" : ""
+  existing_postgresql_database_name     = local.use_existing_db ? "datafold_prod" : ""
+
+  # Other configuration...
+}
+```
+
+## VNet Peering for Cross-VNet Database Access
+
+When your existing PostgreSQL database is located in a different VNet (possibly in a different resource group), the module can automatically set up VNet peering and private endpoints to enable secure connectivity.
+
+### Scenario
+
+This is useful when:
+- Your PostgreSQL database is in a centralized "shared services" VNet
+- The database VNet is managed by a different team or in a different subscription
+- You need private, secure connectivity without exposing the database publicly
+- You want to maintain network isolation while enabling cross-VNet access
+
+### Configuration
+
+In addition to the basic existing database configuration, you need to specify the VNet details:
+
+```hcl
+module "azure" {
+  source = "path/to/terraform-azure-datafold"
+
+  # ... other configuration ...
+
+  # Basic existing database configuration
+  create_database       = true
+  use_existing_database = true
+
+  existing_database_resource_group_name = "shared-services-rg"
+  existing_postgresql_server_name       = "shared-postgres-01"
+  existing_postgresql_database_name     = "datafold_app"
+
+  # VNet peering configuration (required when database is in different VNet)
+  existing_vnet_resource_group_name = "shared-services-network-rg"
+  existing_vnet_name                = "shared-services-vnet"
+  existing_database_subnet_name     = "database-subnet"
+
+  # Optional: Specify existing private DNS zone name (defaults to standard PostgreSQL private link zone)
+  # existing_private_dns_zone_name = "privatelink.postgres.database.azure.com"
+}
+```
+
+### What Gets Created
+
+When using existing database with VNet peering, the module automatically creates:
+
+1. **Bidirectional VNet Peering**:
+   - Peering from your VNet to the existing VNet
+   - Peering from the existing VNet back to your VNet
+
+2. **Private DNS Integration**:
+   - Uses the existing private DNS zone that comes with the existing PostgreSQL server
+   - Links our VNet to the existing private DNS zone for proper name resolution
+
+3. **Private Endpoint**:
+   - Created in your VNet's private endpoint subnet
+   - Points to the existing PostgreSQL server
+   - Automatically integrated with the existing private DNS zone
+
+### Network Flow
+
+```
+Your VNet ←→ VNet Peering ←→ Existing VNet
+    ↓                              ↓
+Private Endpoint  ←---→  PostgreSQL Server
+    ↓                              ↓
+Existing Private DNS Zone ←-------+
+(resolves to private IP)
+```
+
+### Prerequisites
+
+1. **Permissions**: You need contributor access to both resource groups:
+   - Your resource group (for creating resources)
+   - Existing VNet resource group (for creating peering)
+
+2. **No Overlapping CIDRs**: Your VNet CIDR must not overlap with the existing VNet CIDR
+
+3. **PostgreSQL Server**: Must support private endpoints (Azure PostgreSQL Flexible Server) and should already have private DNS integration configured
+
+### Complete Example
+
+```hcl
+locals {
+  environment = "production"
+
+  # Production uses shared database, other environments create their own
+  use_shared_db = local.environment == "production"
+}
+
+module "azure" {
+  source = "path/to/terraform-azure-datafold"
+
+  deployment_name = "datafold-${local.environment}"
+  location        = "East US"
+
+  # VNet configuration - ensure no CIDR overlap with shared services VNet
+  vpc_cidrs = ["10.2.0.0/16"]  # Shared services uses 10.1.0.0/16
+
+  # Database configuration
+  create_database       = true
+  use_existing_database = local.use_shared_db
+
+  # Existing database configuration (production only)
+  existing_database_resource_group_name = local.use_shared_db ? "shared-services-rg" : ""
+  existing_postgresql_server_name       = local.use_shared_db ? "prod-postgres-shared" : ""
+  existing_postgresql_database_name     = local.use_shared_db ? "datafold_prod" : ""
+
+  # VNet peering configuration (production only)
+  existing_vnet_resource_group_name = local.use_shared_db ? "shared-services-network-rg" : ""
+  existing_vnet_name                = local.use_shared_db ? "shared-services-vnet" : ""
+  existing_database_subnet_name     = local.use_shared_db ? "postgres-subnet" : ""
+  existing_private_dns_zone_name    = local.use_shared_db ? "privatelink.postgres.database.azure.com" : ""
+
+  # Other configuration...
+  domain_name = "datafold-${local.environment}.company.com"
+  # ... rest of your configuration
+}
+
+# Access the connection details
+output "postgres_connection" {
+  value = {
+    host     = module.azure.postgres_host          # Resolves to private endpoint when using existing DB
+    database = module.azure.postgres_database_name
+    username = module.azure.postgres_username
+    # Note: password not available for existing databases
+  }
+  sensitive = true
+}
+```
+
+### Troubleshooting
+
+1. **VNet Peering Issues**: Ensure both VNets allow peering and have proper routing
+2. **DNS Resolution**: Verify private DNS zone is linked to both VNets
+3. **Private Endpoint**: Check that the endpoint has a private IP and is connected
+4. **Network Security Groups**: Ensure NSGs allow PostgreSQL traffic (port 5432)
+
+### Outputs
+
+When using existing database with VNet peering, additional outputs are available:
+
+- `postgres_host`: Resolves to the private endpoint FQDN
+- `private_endpoint_ip`: Private IP address of the database endpoint
+- `vnet_peering_status`: Information about the created peering connections

main.tf

@@ -207,6 +207,21 @@ module "database" {
   database_subnet     = module.networking.database_subnet
   private_dns_zone_id = module.networking.database_private_dns_zone_id
 
+  # Existing database configuration
+  use_existing_database                = var.use_existing_database
+  existing_resource_group_name         = var.existing_database_resource_group_name
+  existing_postgresql_server_name      = var.existing_postgresql_server_name
+  existing_postgresql_database_name    = var.existing_postgresql_database_name
+
+  # VNet peering and private endpoint configuration
+  existing_vnet_resource_group_name    = var.existing_vnet_resource_group_name
+  existing_vnet_name                   = var.existing_vnet_name
+  existing_database_subnet_name        = var.existing_database_subnet_name
+  existing_private_dns_zone_name       = var.existing_private_dns_zone_name
+  our_vnet_id                          = module.networking.vpc.id
+  our_vnet_name                        = module.networking.vpc.name
+  our_private_endpoint_subnet_id       = module.networking.private_endpoint_storage_subnet.id
+
   database_username              = var.database_username
   database_name                  = var.database_name
   database_sku                   = var.database_sku
@@ -215,8 +230,10 @@ module "database" {
   postgresql_major_version       = var.postgresql_major_version
 
   # Resource name overrides
-  postgresql_server_name_override   = var.postgresql_server_name_override
-  postgresql_database_name_override = var.postgresql_database_name_override
+  postgresql_server_name_override          = var.postgresql_server_name_override
+  postgresql_database_name_override        = var.postgresql_database_name_override
+  postgresql_private_endpoint_name_override = var.postgresql_private_endpoint_name_override
+  postgresql_vnet_peering_name_prefix_override = var.postgresql_vnet_peering_name_prefix_override
 }
 
 module "clickhouse_backup" {

modules/database/main.tf

@@ -1,10 +1,122 @@
+# Data blocks for existing database resources
+data "azurerm_resource_group" "existing" {
+  count = var.use_existing_database ? 1 : 0
+  name  = var.existing_resource_group_name
+}
+
+data "azurerm_postgresql_flexible_server" "existing" {
+  count               = var.use_existing_database ? 1 : 0
+  name                = var.existing_postgresql_server_name
+  resource_group_name = data.azurerm_resource_group.existing[0].name
+}
+
+# Note: azurerm_postgresql_flexible_server_database data source doesn't exist in provider
+# We'll reference the database name directly from the variable
+
+# Data blocks for existing VNet and networking resources
+data "azurerm_resource_group" "existing_vnet" {
+  count = var.use_existing_database ? 1 : 0
+  name  = var.existing_vnet_resource_group_name
+}
+
+data "azurerm_virtual_network" "existing" {
+  count               = var.use_existing_database ? 1 : 0
+  name                = var.existing_vnet_name
+  resource_group_name = data.azurerm_resource_group.existing_vnet[0].name
+}
+
+data "azurerm_subnet" "existing_database" {
+  count                = var.use_existing_database ? 1 : 0
+  name                 = var.existing_database_subnet_name
+  virtual_network_name = data.azurerm_virtual_network.existing[0].name
+  resource_group_name  = data.azurerm_resource_group.existing_vnet[0].name
+}
+
+# Data block for existing private DNS zone
+data "azurerm_private_dns_zone" "existing_postgresql" {
+  count               = var.use_existing_database ? 1 : 0
+  name                = var.existing_private_dns_zone_name
+  resource_group_name = data.azurerm_resource_group.existing[0].name
+}
+
 locals {
   postgresql_server_name   = var.postgresql_server_name_override != "" ? var.postgresql_server_name_override : "${var.deployment_name}-db-server"
   postgresql_database_name = var.postgresql_database_name_override != "" ? var.postgresql_database_name_override : var.database_name
+  postgresql_private_endpoint_name = var.postgresql_private_endpoint_name_override != "" ? var.postgresql_private_endpoint_name_override : "${var.deployment_name}-postgresql-pe"
+  vnet_peering_name_prefix = var.postgresql_vnet_peering_name_prefix_override != "" ? var.postgresql_vnet_peering_name_prefix_override : var.deployment_name
+}
+
+# VNet Peering from our VNet to existing VNet
+resource "azurerm_virtual_network_peering" "ours_to_existing" {
+  count                     = var.use_existing_database ? 1 : 0
+  name                      = "${local.vnet_peering_name_prefix}-to-${var.existing_vnet_name}"
+  resource_group_name       = var.resource_group_name
+  virtual_network_name      = var.our_vnet_name
+  remote_virtual_network_id = data.azurerm_virtual_network.existing[0].id
+  allow_virtual_network_access = true
+  allow_forwarded_traffic   = true
+  allow_gateway_transit     = false
+  use_remote_gateways       = false
+}
+
+# VNet Peering from existing VNet to our VNet
+resource "azurerm_virtual_network_peering" "existing_to_ours" {
+  count                     = var.use_existing_database ? 1 : 0
+  name                      = "${var.existing_vnet_name}-to-${local.vnet_peering_name_prefix}"
+  resource_group_name       = data.azurerm_resource_group.existing_vnet[0].name
+  virtual_network_name      = data.azurerm_virtual_network.existing[0].name
+  remote_virtual_network_id = var.our_vnet_id
+  allow_virtual_network_access = true
+  allow_forwarded_traffic   = true
+  allow_gateway_transit     = false
+  use_remote_gateways       = false
+}
+
+# Note: Using existing private DNS zone instead of creating a new one
+
+# Link existing private DNS zone to our VNet
+resource "azurerm_private_dns_zone_virtual_network_link" "postgresql_ours" {
+  count                 = var.use_existing_database ? 1 : 0
+  name                  = "${var.deployment_name}-postgresql-dns-link-ours"
+  resource_group_name   = data.azurerm_resource_group.existing[0].name
+  private_dns_zone_name = data.azurerm_private_dns_zone.existing_postgresql[0].name
+  virtual_network_id    = var.our_vnet_id
+  registration_enabled  = false
+
+  tags = var.tags
+}
+
+# Private endpoint in our VNet to connect to existing PostgreSQL server
+resource "azurerm_private_endpoint" "postgresql" {
+  count               = var.use_existing_database ? 1 : 0
+  name                = local.postgresql_private_endpoint_name
+  location            = var.location
+  resource_group_name = var.resource_group_name
+  subnet_id           = var.our_private_endpoint_subnet_id
+
+  private_service_connection {
+    name                           = "${var.deployment_name}-postgresql-psc"
+    private_connection_resource_id = data.azurerm_postgresql_flexible_server.existing[0].id
+    subresource_names              = ["postgresqlServer"]
+    is_manual_connection           = false
+  }
+
+  private_dns_zone_group {
+    name                 = "postgresql-dns-zone-group"
+    private_dns_zone_ids = [data.azurerm_private_dns_zone.existing_postgresql[0].id]
+  }
+
+  tags = var.tags
+
+  depends_on = [
+    azurerm_virtual_network_peering.ours_to_existing,
+    azurerm_virtual_network_peering.existing_to_ours
+  ]
 }
 
 # TODO: Do not hardcode, but create variables for e.g. version, sku_name, etc.
 resource "azurerm_postgresql_flexible_server" "main" {
+  count                         = var.use_existing_database ? 0 : 1
   name                          = local.postgresql_server_name
   resource_group_name           = var.resource_group_name
   location                      = var.location
@@ -13,7 +125,7 @@ resource "azurerm_postgresql_flexible_server" "main" {
   private_dns_zone_id           = var.private_dns_zone_id
   public_network_access_enabled = false
   administrator_login           = var.database_username
-  administrator_password        = random_password.db_password.result
+  administrator_password        = random_password.db_password[0].result
   auto_grow_enabled             = true
   sku_name                      = var.database_sku
   backup_retention_days         = var.database_backup_retention_days
@@ -23,8 +135,9 @@ resource "azurerm_postgresql_flexible_server" "main" {
 }
 
 resource "azurerm_postgresql_flexible_server_database" "main" {
+  count     = var.use_existing_database ? 0 : 1
   name      = local.postgresql_database_name
-  server_id = azurerm_postgresql_flexible_server.main.id
+  server_id = azurerm_postgresql_flexible_server.main[0].id
   collation = "en_US.utf8"
   charset   = "utf8"