Skip to content

Fix backup functionality #2272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 14 commits into
base: develop
Choose a base branch
from
Draft

Fix backup functionality #2272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
a68e9e6
Remove non-functional backup instance for blobs
JimMadge Oct 30, 2024
d618f99
Remove incorrect docs section
JimMadge Nov 12, 2024
fd961a9
Merge remote-tracking branch 'origin/develop' into backup
JimMadge Nov 12, 2024
12dcc65
Remove old backup infrastructure
JimMadge Nov 12, 2024
ceb302a
WIP: Add backup VM
JimMadge Nov 12, 2024
0bead11
Merge remote-tracking branch 'origin/develop' into backup
JimMadge Dec 9, 2024
32f5eba
Add backup subnet
JimMadge Jan 20, 2025
fbdbcfe
Configure backup nsg
JimMadge Jan 20, 2025
f6b9a14
Merge remote-tracking branch 'origin/develop' into backup
JimMadge Jan 20, 2025
85d4154
Correct nsg rules
JimMadge Jan 20, 2025
cc6af27
Run lint:fmt
JimMadge Feb 6, 2025
f465459
Correct arguments
JimMadge Feb 6, 2025
aa10d69
Update subnet ranges in tests
JimMadge Feb 6, 2025
56eb224
Merge remote-tracking branch 'origin/develop' into backup
JimMadge Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions data_safe_haven/infrastructure/common/ip_ranges.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class SREIpRanges:
vnet = AzureIPv4Range("10.0.0.0", "10.0.255.255")
application_gateway = vnet.next_subnet(256)
apt_proxy_server = vnet.next_subnet(8)
backup = vnet.next_subnet(8)
clamav_mirror = vnet.next_subnet(8)
data_configuration = vnet.next_subnet(8)
data_private = vnet.next_subnet(8)
Expand Down
13 changes: 11 additions & 2 deletions data_safe_haven/infrastructure/programs/declarative_sre.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,9 +401,9 @@ def __call__(self) -> None:
maintenance_configuration_id=monitoring.maintenance_configuration.id,
resource_group_name=resource_group.name,
sre_name=self.config.name,
storage_account_desired_state_name=desired_state.storage_account_name,
storage_account_data_private_user_name=data.storage_account_data_private_user_name,
storage_account_data_private_sensitive_name=data.storage_account_data_private_sensitive_name,
storage_account_desired_state_name=desired_state.storage_account_name,
subnet_workspaces=networking.subnet_workspaces,
subscription_name=sre_subscription_name,
virtual_network=networking.virtual_network,
Expand All @@ -418,10 +418,19 @@ def __call__(self) -> None:
"sre_backup",
self.stack_name,
SREBackupProps(
admin_password=...,
admin_username=...,
apt_proxy_server_hostname=apt_proxy_server.hostname,
data_collection_rule_id=monitoring.data_collection_rule_vms.id,
data_collection_endpoint_id=monitoring.data_collection_endpoint.id,
location=self.config.azure.location,
maintenance_configuration_id=monitoring.maintenance_configuration.id,
resource_group_name=resource_group.name,
storage_account_data_private_sensitive_id=data.storage_account_data_private_sensitive_id,
storage_account_data_private_user_name=data.storage_account_data_private_user_name,
storage_account_data_private_sensitive_name=data.storage_account_data_private_sensitive_name,
storage_account_desired_state_name=desired_state.storage_account_name,
subnet_backup=networking.subnet_backup,
virtual_network=networking.virtual_network,
),
tags=self.tags,
)
Expand Down
216 changes: 63 additions & 153 deletions data_safe_haven/infrastructure/programs/sre/backup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,57 @@

from collections.abc import Mapping

from pulumi import ComponentResource, Input, ResourceOptions
from pulumi_azure_native import dataprotection
from pulumi import ComponentResource, Input, Output, ResourceOptions
from pulumi_azure_native import network

from data_safe_haven.functions import b64encode, replace_separators
from data_safe_haven.infrastructure.common import (
get_name_from_subnet,
get_name_from_vnet,
)
from data_safe_haven.infrastructure.components import LinuxVMComponentProps, VMComponent


class SREBackupProps:
"""Properties for SREBackupComponent"""

def __init__(
self,
admin_password: Input[str],
admin_username: Input[str],
apt_proxy_server_hostname: Input[str],
data_collection_rule_id: Input[str],
data_collection_endpoint_id: Input[str],
location: Input[str],
maintenance_configuration_id: Input[str],
resource_group_name: Input[str],
storage_account_data_private_sensitive_id: Input[str],
storage_account_data_private_sensitive_name: Input[str],
storage_account_data_private_user_name: Input[str],
storage_account_desired_state_name: Input[str],
subnet_backup: Input[network.GetSubnetResult],
virtual_network: Input[network.VirtualNetwork],
) -> None:
self.admin_password = admin_password
self.admin_username = admin_username
self.apt_proxy_server_hostname = apt_proxy_server_hostname
self.data_collection_rule_id = data_collection_rule_id
self.data_collection_endpoint_id = data_collection_endpoint_id
self.location = location
self.maintenance_configuration_id = maintenance_configuration_id
self.resource_group_name = resource_group_name
self.storage_account_data_private_sensitive_id = (
storage_account_data_private_sensitive_id
)
self.storage_account_data_private_sensitive_name = (
storage_account_data_private_sensitive_name
)
self.storage_account_data_private_user_name = (
storage_account_data_private_user_name
)
self.storage_account_desired_state_name = storage_account_desired_state_name
self.subnet_backup_name = Output.from_input(subnet_backup).apply(
get_name_from_subnet
)
self.virtual_network_name = Output.from_input(virtual_network).apply(
get_name_from_vnet
)


class SREBackupComponent(ComponentResource):
Expand All @@ -41,154 +70,35 @@ def __init__(
child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self))
child_tags = {"component": "backup"} | (tags if tags else {})

# Deploy backup vault
backup_vault = dataprotection.BackupVault(
f"{self._name}_backup_vault",
identity=dataprotection.DppIdentityDetailsArgs(
type="SystemAssigned",
),
location=props.location,
properties=dataprotection.BackupVaultArgs(
storage_settings=[
dataprotection.StorageSettingArgs(
datastore_type=dataprotection.StorageSettingStoreTypes.VAULT_STORE,
type=dataprotection.StorageSettingTypes.LOCALLY_REDUNDANT,
)
],
),
resource_group_name=props.resource_group_name,
vault_name=f"{stack_name}-bv-backup",
opts=child_opts,
tags=child_tags,
)
# Template cloud init
cloudinit = Output.all(
apt_proxy_server_hostname=props.apt_proxy_server_hostname,
storage_account_desired_state_name=props.storage_account_desired_state_name,
storage_account_data_private_user_name=props.storage_account_data_private_user_name,
storage_account_data_private_sensitive_name=props.storage_account_data_private_sensitive_name,
).apply(lambda kwargs: self.template_cloudinit(**kwargs))

# Backup policy for blobs
backup_policy_blobs = dataprotection.BackupPolicy(
f"{self._name}_backup_policy_blobs",
backup_policy_name="backup-policy-blobs",
properties=dataprotection.BackupPolicyArgs(
datasource_types=["Microsoft.Storage/storageAccounts/blobServices"],
object_type="BackupPolicy",
policy_rules=[
# Retain for 30 days
dataprotection.AzureRetentionRuleArgs(
is_default=True,
lifecycles=[
dataprotection.SourceLifeCycleArgs(
delete_after=dataprotection.AbsoluteDeleteOptionArgs(
duration="P30D",
object_type="AbsoluteDeleteOption",
),
source_data_store=dataprotection.DataStoreInfoBaseArgs(
data_store_type=dataprotection.DataStoreTypes.OPERATIONAL_STORE,
object_type="DataStoreInfoBase",
),
),
],
name="Default",
object_type="AzureRetentionRule",
),
],
),
resource_group_name=props.resource_group_name,
vault_name=backup_vault.name,
opts=ResourceOptions.merge(
child_opts, ResourceOptions(parent=backup_vault)
),
)

# Backup policy for disks
dataprotection.BackupPolicy(
f"{self._name}_backup_policy_disks",
backup_policy_name="backup-policy-disks",
properties=dataprotection.BackupPolicyArgs(
datasource_types=["Microsoft.Compute/disks"],
object_type="BackupPolicy",
policy_rules=[
# Backup at 02:00 every day
dataprotection.AzureBackupRuleArgs(
backup_parameters=dataprotection.AzureBackupParamsArgs(
backup_type="Incremental",
object_type="AzureBackupParams",
),
data_store=dataprotection.DataStoreInfoBaseArgs(
data_store_type=dataprotection.DataStoreTypes.OPERATIONAL_STORE,
object_type="DataStoreInfoBase",
),
name="BackupDaily",
object_type="AzureBackupRule",
trigger=dataprotection.ScheduleBasedTriggerContextArgs(
object_type="ScheduleBasedTriggerContext",
schedule=dataprotection.BackupScheduleArgs(
repeating_time_intervals=[
"R/2023-01-01T02:00:00+00:00/P1D"
],
),
tagging_criteria=[
dataprotection.TaggingCriteriaArgs(
is_default=True,
tag_info=dataprotection.RetentionTagArgs(
tag_name="Default",
),
tagging_priority=99,
)
],
),
),
# Retain for 30 days
dataprotection.AzureRetentionRuleArgs(
is_default=True,
lifecycles=[
dataprotection.SourceLifeCycleArgs(
delete_after=dataprotection.AbsoluteDeleteOptionArgs(
duration="P30D",
object_type="AbsoluteDeleteOption",
),
source_data_store=dataprotection.DataStoreInfoBaseArgs(
data_store_type=dataprotection.DataStoreTypes.OPERATIONAL_STORE,
object_type="DataStoreInfoBase",
),
),
],
name="Default",
object_type="AzureRetentionRule",
),
],
),
resource_group_name=props.resource_group_name,
vault_name=backup_vault.name,
opts=ResourceOptions.merge(
child_opts, ResourceOptions(parent=backup_vault)
),
)

# Backup instance for blobs
dataprotection.BackupInstance(
f"{self._name}_backup_instance_blobs",
backup_instance_name="backup-instance-blobs",
properties=dataprotection.BackupInstanceArgs(
data_source_info=dataprotection.DatasourceArgs(
resource_id=props.storage_account_data_private_sensitive_id,
datasource_type="Microsoft.Storage/storageAccounts/blobServices",
object_type="Datasource",
resource_location=props.location,
resource_name=props.storage_account_data_private_sensitive_name,
resource_type="Microsoft.Storage/storageAccounts",
resource_uri=props.storage_account_data_private_sensitive_id,
# Backup virtual machine
VMComponent(
replace_separators(f"{self._name}_vm_backup", "_"),
LinuxVMComponentProps(
admin_password=props.admin_password,
admin_username=props.admin_username,
b64cloudinit=cloudinit.apply(b64encode),
data_collection_rule_id=props.data_collection_rule_id,
data_collection_endpoint_id=props.data_collection_endpoint_id,
ip_address_private=...,
location=props.location,
maintenance_configuration_id=props.maintenance_configuration_id,
resource_group_name=props.resource_group_name,
subnet_name=props.subnet_backup_name,
virtual_network_name=props.virtual_network_name,
virtual_network_resource_group_name=props.resource_group_name,
vm_name=Output.concat(stack_name, "-vm-backup").apply(
lambda s: replace_separators(s, "-")
),
object_type="BackupInstance",
policy_info=dataprotection.PolicyInfoArgs(
policy_id=backup_policy_blobs.id,
),
friendly_name="BlobBackupSensitiveData",
),
resource_group_name=props.resource_group_name,
vault_name=backup_vault.name,
opts=ResourceOptions.merge(
child_opts, ResourceOptions(parent=backup_policy_blobs)
vm_size="Standard_B2s_v2",
),
opts=child_opts,
tags=child_tags,
)

# Backup instance for disks
# We currently have no disks except OS disks so no backup is needed
# This may change in future, so we leave the policy above
Loading
Loading