template.yaml

AWSTemplateFormatVersion: '2010-09-09'
Transform:
  - AWS::Serverless-2016-10-31
Description: >
  canvas-data-2

  This template contains infrastructure for maintaining a Postgresql database containing
  data sourced from Canvas Data 2. It uses the higher-level `dap` functionality from
  instructure to maintain the local tables.

  A Step Function is used to orchestrate the synchronization operations.

Parameters:

  EnvironmentParameter:
    Type: String
    Description: Name of the environment. Allowed values are dev, stg, prod.
    AllowedValues:
      - "dev"
      - "stg"
      - "prod"
    Default: "dev"

  LogLevel:
    Type: String
    Description: Logging level. Allowed values are DEBUG, INFO, WARN, ERROR.
    AllowedValues:
      - 'DEBUG'
      - 'INFO'
      - 'WARN'
      - 'ERROR'
    Default: INFO

  SkipTablesParameter:
    Type: String
    Description: List of Canvas Data 2 tables not to initialize/sync. Leave empty to load all tables.

  VpcIdParameter:
    Type: AWS::EC2::VPC::Id
    Description: The ID of the VPC to deploy the database and Lambda functions into.

  LambdaSubnetListParameter:
    Type: List<AWS::EC2::Subnet::Id>
    Description: The list of one or more subnets to deploy the Lambda functions into.

  DatabaseSubnetListParameter:
    Type: List<AWS::EC2::Subnet::Id>
    Description: The list of one or more subnets to deploy the database into.

  SsmPathParameter:
    Type: String
    Description: Path to DAP client ID and secret.
    Default: canvas_data_2

  DatabaseAdminUserParameter:
    Type: String
    Description: Username for the admin database user.

  DatabaseCd2UserParameter:
    Type: String
    Description: Username for the canvas database user.

  DatabaseMinCapacityParameter:
    Type: Number
    Description: Serverless Aurora minimum capacity.
    Default: 0.5

  DatabaseMaxCapacityParameter:
    Type: Number
    Description: Serverless Aurora maximum capacity.
    Default: 4

  # Task sizing:
  # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size
  TaskCpuParameter:
    Type: Number
    Description: CPUs allocted to Fargate tasks.
    Default: 1024

  TaskMemoryParameter:
    Type: Number
    Description: Memory allocated to Fargate tasks (MB).
    Default: 8192

  TaskStorageParameter:
    Type: Number
    Description: Ephemeral storage allocated to Fargate tasks (GB).
    Default: 80

  ResourcePrefixParameter:
    Type: String
    Description: Prefix for resource names.

  TagNameParameter:
    Type: String
    Description: Cost tag name.

  TagValueParameter:
    Type: String
    Description: Cost tag value.

  EcrAccountNumberParameter:
    Type: String
    Description: AWS Account ID for an ECR repository.

  LogRetentionParameter:
    Type: Number
    Description: Number of days to retain logs.
    Default: 90

  YourRoleArnParameter:
    Type: String
    Description: The ARN for the IAM role creating/updating this stack.

  DatabaseClusterArnParameter:
    Type: String
    Description: (Optional) An existing Aurora database cluster ARN. Leave empty to create a new Aurora database cluster.
    Default: ''

  # Default value is an ARN because it's referenced in the stack within a conditional statement.
  # It must resolve as an ARN for the template to be valid.
  DatabaseAdminSecretArnParameter:
    Type: String
    Description: (Optional) An existing database admin secret ARN. Use default value if DatabaseClusterArnParameter is empty.
    Default: arn:aws:secretsmanager:region:123456789123:secret:placeholder

  DatabaseSecurityGroupParameter:
    Type: String
    Description: (Optional) A security group ID for the database. Leave empty to create a new security group.
    Default: ''

  DatabaseClientSecurityGroupParameter:
    Type: String
    Description: (Optional) A security group ID for the DatabaseClient. Leave empty to create a new security group.
    Default: ''

  ListTablesFunctionSecurityGroupParameter:
    Type: String
    Description: (Optional) A security group ID for the ListTables Lambda function. Leave empty to create a new security group.
    Default: ''

  AthenaConnectorParameter:
    Type: String
    Description: (Optional) Create an Athena connector for PostgreSQL. Default is false. Must be false when values are set for DatabaseAdminSecretArnParameter and DatabaseSecurityGroupParameter.
    AllowedValues: [true, false]
    Default: false

  AthenaConnectorLambdaNameParameter:
    Type: String
    Description: (Optional) Name of Athena Connector Lambda. Unused if AthenaConnectorParameter is set to false.
    Default: athena_postgresql_connector_cd2

  NotificationTopicParameter:
    Type: String
    Description: (Optional) SNS topic to send notifications. Leave empty to create a new topic.
    Default: ''

  FalconSensorParameter:
    Type: String
    Description: (Optional) Configure CrowdStrike Falcon container sensor. Default is false.
    AllowedValues: [true, false]
    Default: false

  FalconSensorRepoParameter:
    Type: String
    Description: (Optional) Repo for CrowdStrike Falcon container sensor image. Leave empty if FalconSensorParameter is false.
    Default: ''

  FalconSensorOptionsParameter:
    Type: String
    Description: (Optional) SSM Parameter Store key for Falcon sensor options. Leave empty if FalconSensorParameter is false.
    Default: ''

  SlackWebHookURLSecretNameParameter:
    Type: String
    Description: The secret name for Slack Incoming Webhook URL for the notification
    Default: ''

  SecretsKmsKeyIDParameter:
    Type: String
    Description: (Optional) The key ID for existing SecretsKmsKey. Leave empty to create a new KMS key.
    Default: ''

  DataKmsKeyIDParameter:
    Type: String
    Description: (Optional) The Key ID for DataKmsKey. Leave empty to create a new KMS key.
    Default: ''

  MainCD2StackNameParameter:
    Type: String
    Description: (Optional) The Cloudformation stack name of the main Canvas Data 2 Stack.
    Default: 'ubcla-canvas-data-2'

  NamespaceImportListParameter:
    Type: String
    Description: Comma delimited string to list Canvas namespaces to import
    Default: "canvas"

Conditions:

  # Conditions for Bring Your Own Aurora Database Cluster
  # Requires both DatabaseClusterArnParameter and DatabaseAdminSecretArnParameter to be specified
  ExistingDatabase: !Not [!Equals [!Ref DatabaseClusterArnParameter, '']]
  ExistingDatabaseAdminSecret: !Not [!Equals [!Ref DatabaseAdminSecretArnParameter, 'arn:aws:secretsmanager:region:123456789123:secret:placeholder']]
  CreateDatabase: !Not
    - !Or
        - Condition: ExistingDatabase
        - Condition: ExistingDatabaseAdminSecret

  # Conditions for Bring Your Own Data Security Group
  ExistingDatabaseSecurityGroup: !Not [!Equals [!Ref DatabaseSecurityGroupParameter, '']]
  CreateDatabaseSecurityGroup: !Equals [!Ref DatabaseSecurityGroupParameter, '']

  # Conditions for Bring Your Own Database Client Security Group
  ExistingDatabaseClientSecurityGroup: !Not [!Equals [!Ref DatabaseClientSecurityGroupParameter, '']]
  CreateDatabaseClientSecurityGroup: !Equals [!Ref DatabaseClientSecurityGroupParameter, '']

  # Conditions for Bring Your Own ListTables Lambda Function Security Group
  ExistingListTablesFunctionSecurityGroup: !Not [!Equals [!Ref ListTablesFunctionSecurityGroupParameter, '']]
  CreateListTablesFunctionSecurityGroup: !Equals [!Ref ListTablesFunctionSecurityGroupParameter, '']

  # Conditions for Bring Your Own Notification Topic
  ExistingNotificationTopic: !Not [!Equals [!Ref NotificationTopicParameter, '']]
  CreateNotificationTopic: !Equals [!Ref NotificationTopicParameter, '']

  # Condition for optional CrowdStrike Falcon sensor
  # Configures an additional container task that injects the Falcon sensor into Fargate tasks
  # For more info: https://github.com/CrowdStrike/Container-Security/blob/0d2ae005be852aae403df3feb11e3bf2fd1f5258/aws-ecs/ecs-fargate-guide.md
  ConfigureFalconSensor: !Equals [true, !Ref FalconSensorParameter]

  # Conditions for optional Athena connector
  # Creates an Athena data source allowing CD2 data to be joined with other data sources
  # Requires CreateDatabase condition to be true to ensure cluster endpoint and port exist
  AthenaConnector: !Equals [true, !Ref AthenaConnectorParameter]
  CreatePrimaryCanvasAthenaConnector: !And
    - Condition: AthenaConnector
    - Condition: CreateDatabase

  IsCatalogNamespaceIncluded: !Or
    - !Equals [ !Ref NamespaceImportListParameter, "catalog" ]
    - !Equals [ !Ref NamespaceImportListParameter, "canvas,catalog" ]
    - !Equals [ !Ref NamespaceImportListParameter, "catalog,canvas" ]

  CreateAdditionalPrimaryCanvasAthenaConnector: !And
    - Condition: ExistingDatabase
    - Condition: AthenaConnector

  CreateAthenaConnector: !Or
      - Condition: CreatePrimaryCanvasAthenaConnector
      - Condition: CreateAdditionalPrimaryCanvasAthenaConnector

  CreateCatalogAthenaConnector: !And
    - Condition: ExistingDatabase
    - Condition: IsCatalogNamespaceIncluded

Resources:

  SecretsKmsKey:
    Type: AWS::KMS::Key
    Condition: CreateDatabase
    Properties:
      Description: Key for encrypting Canvas Data 2 secrets
      Enabled: true
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  SecretsKeyAlias:
    Type: AWS::KMS::Alias
    Condition: CreateDatabase
    Properties:
      AliasName: !Sub alias/${AWS::StackName}-cd2-secrets
      TargetKeyId: !Ref SecretsKmsKey

  DataKmsKey:
    Type: AWS::KMS::Key
    Condition: CreateDatabase
    Properties:
      Description: Key for encrypting Canvas Data 2 data
      Enabled: true
      KeyPolicy:
        Version: '2012-10-17'
        Statement:
        - Sid: Allow root account admistration of key
          Effect: Allow
          Principal:
            AWS: !Sub arn:aws:iam::${AWS::AccountId}:root
          Action: kms:*
          Resource: '*'
        - Sid: Allow generate data key access for Fargate tasks.
          Effect: Allow
          Principal:
            Service: fargate.amazonaws.com
          Action:
            - kms:GenerateDataKeyWithoutPlaintext
          Condition:
            StringEquals:
              "kms:EncryptionContext:aws:ecs:clusterAccount":
                - !Sub ${AWS::AccountId}
            ForAnyValue:StringEquals:
              "kms:EncryptionContext:aws:ecs:clusterName":
                - !Sub ${AWS::StackName}-cd2-cluster
                - !Sub ${AWS::StackName}-catalog-cd2-cluster
          Resource: '*'
        - Sid: Allow grant creation permission for Fargate tasks.
          Effect: Allow
          Principal:
            Service: fargate.amazonaws.com
          Action:
            - kms:CreateGrant
          Condition:
            StringEquals:
              "kms:EncryptionContext:aws:ecs:clusterAccount":
                - !Sub ${AWS::AccountId}
            ForAnyValue:StringEquals:
              "kms:EncryptionContext:aws:ecs:clusterName":
                - !Sub ${AWS::StackName}-cd2-cluster
                - !Sub ${AWS::StackName}-catalog-cd2-cluster
            ForAllValues:StringEquals:
              "kms:GrantOperations":
                - "Decrypt"
          Resource: '*'
        - Sid: Allow describe key permission for cluster operator - CreateCluster and UpdateCluster.
          Effect: Allow
          Principal:
            AWS: !Ref YourRoleArnParameter
          Action:
            - kms:DescribeKey
          Resource: '*'
        - Sid: Allow RDS to use the key to encrypt the cluster
          Effect: Allow
          Principal:
            Service: rds.amazonaws.com
          Action:
            - kms:Encrypt
            - kms:Decrypt
            - kms:ReEncrypt*
            - kms:GenerateDataKey*
            - kms:DescribeKey
          Resource: '*'
          # Removing condition to avoid circular dependency
          # Condition:
          #   StringEquals:
          #     "rds:db-cluster-id": !Sub ${AuroraDatabaseCluster}
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  DataKeyAlias:
    Type: AWS::KMS::Alias
    Condition: CreateDatabase
    Properties:
      AliasName: !Sub alias/${AWS::StackName}-cd2-data
      TargetKeyId: !Ref DataKmsKey

  # Create a secret for the CD2 app "canvas" user
  DatabaseUserSecretCanvas:
    Type: AWS::SecretsManager::Secret
    Properties:
      Name: !Sub ${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-${DatabaseCd2UserParameter}
      Description: Database user for Canvas Data 2 canvas user
      GenerateSecretString:
        SecretStringTemplate: !Sub '{"username": "${DatabaseCd2UserParameter}"}'
        GenerateStringKey: password
        PasswordLength: 24
        ExcludePunctuation: true
      KmsKeyId: !If [CreateDatabase, !Ref SecretsKmsKey, !Ref SecretsKmsKeyIDParameter]
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  # Attach Canvas database user Secret to the cluster
  DatabaseUserSecretAttachmentCanvas:
    Type: AWS::SecretsManager::SecretTargetAttachment
    Properties:
      SecretId: !Ref DatabaseUserSecretCanvas
      TargetId: !If [ExistingDatabase, !Select [ "6", !Split [ ":" , !Ref DatabaseClusterArnParameter ] ], !Ref AuroraDatabaseCluster]
      TargetType: AWS::RDS::DBCluster

  DatabaseSubnetGroup:
    Type: AWS::RDS::DBSubnetGroup
    Condition: CreateDatabase
    Properties:
      DBSubnetGroupDescription: !Sub ${ResourcePrefixParameter}-cd2-subnetgroup-${EnvironmentParameter}
      SubnetIds: !Ref DatabaseSubnetListParameter
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  DatabaseClientSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Condition: CreateDatabaseClientSecurityGroup
    Properties:
      GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-client-sg-${EnvironmentParameter}
      GroupName: Canvas Data 2 Database Client
      VpcId: !Ref VpcIdParameter
      SecurityGroupEgress:
        - IpProtocol: tcp
          FromPort: 443
          ToPort: 443
          CidrIp: 0.0.0.0/0
      Tags:
        - Key: Name
          Value: Canvas Data 2 Database Client
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  ListTablesFunctionSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Condition: CreateListTablesFunctionSecurityGroup
    Properties:
      GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-list-tables-function-sg-${EnvironmentParameter}
      GroupName: Canvas Data 2 ListTablesFunction
      VpcId: !Ref VpcIdParameter
      SecurityGroupEgress:
        - IpProtocol: tcp
          FromPort: 443
          ToPort: 443
          CidrIp: 0.0.0.0/0
      Tags:
        - Key: Name
          Value: Canvas Data 2 ListTablesFunction
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  DatabaseSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Condition: CreateDatabaseSecurityGroup
    Properties:
      GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-sg-${EnvironmentParameter}
      GroupName: Canvas Data 2 Database
      VpcId: !Ref VpcIdParameter
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 5432
          ToPort: 5432
          SourceSecurityGroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup]
        # TODO: add more ingress rules based on parameter(s)
      Tags:
        - Key: Name
          Value: Canvas Data 2 Database
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  DatabaseClientEgressToDatabase:
    Type: AWS::EC2::SecurityGroupEgress
    Condition: CreateDatabaseSecurityGroup
    Properties:
      GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup]
      IpProtocol: tcp
      FromPort: 5432
      ToPort: 5432
      DestinationSecurityGroupId: !GetAtt DatabaseSecurityGroup.GroupId

  DatabaseClientEgressToExistingDatabase:
    Type: AWS::EC2::SecurityGroupEgress
    Condition: CreateDatabaseClientSecurityGroup
    Properties:
      GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup]
      IpProtocol: tcp
      FromPort: 5432
      ToPort: 5432
      DestinationSecurityGroupId: !Ref DatabaseSecurityGroupParameter

  AuroraDatabaseCluster:
    Type: AWS::RDS::DBCluster
    Condition: CreateDatabase
    Properties:
      Engine: aurora-postgresql
      EngineVersion: 15.5
      MasterUsername: !Sub ${DatabaseAdminUserParameter}
      ManageMasterUserPassword: true
      MasterUserSecret:
        KmsKeyId: !Ref SecretsKmsKey
      DBSubnetGroupName: !Ref DatabaseSubnetGroup
      VpcSecurityGroupIds:
          - !If [ExistingDatabaseSecurityGroup, !Ref DatabaseSecurityGroupParameter, !Ref DatabaseSecurityGroup]
      Port: 5432
      DatabaseName: cd2
      ServerlessV2ScalingConfiguration:
        MinCapacity: !Ref DatabaseMinCapacityParameter
        MaxCapacity: !Ref DatabaseMaxCapacityParameter
      DeletionProtection: false
      EnableHttpEndpoint: true
      BackupRetentionPeriod: 7
      StorageEncrypted: true
      KmsKeyId: !GetAtt DataKmsKey.Arn
      DBClusterParameterGroupName: !Ref DatabaseClusterParameterGroup
      EnableCloudwatchLogsExports:
        - postgresql
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  DatabaseClusterParameterGroup:
    Type: AWS::RDS::DBClusterParameterGroup
    Properties:
      Description: PostgreSQL parameter group for Canvas Data 2
      Family: aurora-postgresql15
      Parameters:
        rds.force_ssl: 1
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  AuroraDatabaseInstance:
    Type: AWS::RDS::DBInstance
    Condition: CreateDatabase
    Properties:
      Engine: aurora-postgresql
      DBInstanceClass: db.serverless
      DBClusterIdentifier: !Ref AuroraDatabaseCluster
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  DatabasePostgresqlLogGroup:
    Type: AWS::Logs::LogGroup
    Condition: CreateDatabase
    Properties:
      LogGroupName: !Sub /aws/rds/cluster/${AuroraDatabaseCluster}/postgresql
      RetentionInDays: !Ref LogRetentionParameter
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  ListTablesFunction:
    Type: AWS::Serverless::Function
    Properties:
      CodeUri: list_tables/
      Handler: app.lambda_handler
      Runtime: python3.11
      Architectures:
        - x86_64
      Policies:
        - SSMParameterReadPolicy:
            ParameterName: !Sub '${EnvironmentParameter}/${SsmPathParameter}*'
      Environment:
        Variables:
          ENV: !Ref EnvironmentParameter
          POWERTOOLS_METRICS_NAMESPACE: canvas-data-2
          POWERTOOLS_SERVICE_NAME: list_tables
          LOG_LEVEL: !Ref LogLevel
          SKIP_TABLES: !Ref SkipTablesParameter
          SLACK_WEBHOOK_SECRET_NAME: !Sub ${SlackWebHookURLSecretNameParameter}
          STACK_NAME: !Sub ${AWS::StackName}
          SSM_PARAMETER_NAME: !Sub ${SsmPathParameter}
          DB_CD2_USER: !Sub ${DatabaseCd2UserParameter}
      Timeout: 120
      MemorySize: 256
      VpcConfig:
        SecurityGroupIds:
          - !If [ExistingListTablesFunctionSecurityGroup, !Ref ListTablesFunctionSecurityGroupParameter, !Ref ListTablesFunctionSecurityGroup]
        SubnetIds: !Ref LambdaSubnetListParameter
      LoggingConfig:
        LogGroup: !Ref ListTablesLogGroup
      Role: !GetAtt ListTablesRole.Arn
      Layers:
        - !Ref LambdaLayer

  ListTablesLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub ${AWS::StackName}/lambda/ListTables
      RetentionInDays: !Ref LogRetentionParameter
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  ListTablesRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: lambda.amazonaws.com
          Action: sts:AssumeRole
      Path: '/'
      Policies:
      - PolicyName: parameter-store
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - ssm:DescribeParameters
            Resource: '*'
          - Effect: Allow
            Action:
            - ssm:GetParameters
            - ssm:GetParameter
            - ssm:GetParametersByPath
            Resource:
            - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${EnvironmentParameter}/${SsmPathParameter}*
      - PolicyName: SecretsManagerPolicy
        PolicyDocument:
          Version: "2012-10-17"
          Statement:
            - Effect: Allow
              Action:
                - secretsmanager:GetSecretValue
              Resource: !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SlackWebHookURLSecretNameParameter}*
      - PolicyName: logging
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - logs:CreateLogGroup
              Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*
            - Effect: Allow
              Action:
              - logs:CreateLogStream
              - logs:PutLogEvents
              Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/${AWS::StackName}/lambda/ListTables*
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole

  FargateCluster:
    Type: AWS::ECS::Cluster
    Properties:
      ClusterName: !Sub ${AWS::StackName}-cd2-cluster
      Configuration:
        ManagedStorageConfiguration:
          FargateEphemeralStorageKmsKeyId: !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${DataKmsKeyIDParameter}"]
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  FargateExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Action:
              - sts:AssumeRole
            Principal:
              Service:
                - ecs-tasks.amazonaws.com
                - events.amazonaws.com
      Path: '/'
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      Policies:
        - PolicyName: iam_pass_role
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - iam:PassRole
                Resource:
                  - !Sub ${TaskRole.Arn}
        - PolicyName: kms_for_data
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - kms:*
                Resource:
                  - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${DataKmsKeyIDParameter}"]
        - !If
          - ConfigureFalconSensor
          - PolicyName: falcon_parameter_store
            PolicyDocument:
              Version: '2012-10-17'
              Statement:
                - Effect: Allow
                  Action:
                    - ssm:DescribeParameters
                  Resource: '*'
                - Effect: Allow
                  Action:
                    - ssm:GetParameters
                    - ssm:GetParameter
                    - ssm:GetParametersByPath
                  Resource:
                    - !Sub "arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter${FalconSensorOptionsParameter}*"
          - !Ref "AWS::NoValue"

  TaskRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Action:
          - sts:AssumeRole
          Principal:
            Service:
            - ecs-tasks.amazonaws.com
            - events.amazonaws.com
      Path: '/'
      Policies:
      - PolicyName: parameter-store
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - ssm:DescribeParameters
            Resource: '*'
          - Effect: Allow
            Action:
            - ssm:GetParameters
            - ssm:GetParameter
            - ssm:GetParametersByPath
            Resource:
            - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${EnvironmentParameter}/${SsmPathParameter}*
            - !If [ConfigureFalconSensor, !Sub "arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter${FalconSensorOptionsParameter}*", !Ref "AWS::NoValue"]
      - PolicyName: secrets
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - secretsmanager:GetSecretValue
            Resource:
            - !Ref DatabaseUserSecretCanvas
            - !If [ExistingDatabaseAdminSecret, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn]
      - PolicyName: kms
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - kms:Decrypt
            Resource: !If [CreateDatabase, !GetAtt SecretsKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${SecretsKmsKeyIDParameter}"]
      - PolicyName: ecr
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - ecr:BatchCheckLayerAvailability
            - ecr:GetDownloadUrlForLayer
            - ecr:BatchGetImage
            - ecr:DescribeRepositories
            - ecr:DescribeImages
            - ecr:GetAuthorizationToken
            Resource:
            - !Sub arn:aws:ecr:${AWS::Region}:${EcrAccountNumberParameter}:repository/*
      - PolicyName: rds_data
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - rds-data:*
            Resource:
            - !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn]
      - PolicyName: state_machine
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action:
            - states:SendTaskSuccess
            - states:SendTaskFailure
            # An explicit reference to the CD2 State Machine creates a CloudFormation circular dependency
            # This isn't a great work-around.
            Resource:
            - '*'
      - PolicyName: kms_for_data
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - kms:*
              Resource:
                - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${DataKmsKeyIDParameter}"]
      - PolicyName: logging
        PolicyDocument:
          Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - logs:CreateLogGroup
              Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*
            - Effect: Allow
              Action:
              - logs:CreateLogStream
              - logs:PutLogEvents
              Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/${AWS::StackName}/fargate/*
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy

  InitTableTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family: !Sub ${AWS::StackName}-InitTable
      Cpu: !Ref TaskCpuParameter
      Memory: !Ref TaskMemoryParameter
      EphemeralStorage:
        SizeInGiB: !Ref TaskStorageParameter
      NetworkMode: awsvpc
      RequiresCompatibilities:
        - FARGATE
      ExecutionRoleArn: !Sub ${FargateExecutionRole.Arn}
      TaskRoleArn: !Sub ${TaskRole.Arn}
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}
      ContainerDefinitions:
        - Name: !Sub ${AWS::StackName}-InitTable
          Cpu: !Ref TaskCpuParameter
          Memory: !Ref TaskMemoryParameter
          Image:
            Fn::If:
              - CreateDatabase
              - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}/init-table:${EnvironmentParameter}
              - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${MainCD2StackNameParameter}/init-table:${EnvironmentParameter}
          Essential: true
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref InitTableLogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: InitTable
          # Additional configuration for Falcon container sensor
          EntryPoint:
            !If
              - ConfigureFalconSensor
              -
                - /tmp/CrowdStrike/rootfs/lib64/ld-linux-x86-64.so.2
                - --library-path
                - /tmp/CrowdStrike/rootfs/lib64
                - /tmp/CrowdStrike/rootfs/bin/bash
                - /tmp/CrowdStrike/rootfs/entrypoint-ecs.sh
                - python
                - app.py
              - !Ref "AWS::NoValue"
          Secrets:
            !If
              - ConfigureFalconSensor
              -
                - Name: FALCONCTL_OPTS
                  ValueFrom: !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FalconSensorOptionsParameter}
              - !Ref "AWS::NoValue"
          LinuxParameters:
            !If
              - ConfigureFalconSensor
              - Capabilities:
                  Add:
                    - SYS_PTRACE
              - !Ref "AWS::NoValue"
          MountPoints:
            !If
              - ConfigureFalconSensor
              -
                - ContainerPath: /tmp/CrowdStrike
                  SourceVolume: crowdstrike-falcon-volume
                  ReadOnly: true
              - !Ref "AWS::NoValue"
          DependsOn:
            !If
              - ConfigureFalconSensor
              -
                - Condition: COMPLETE
                  ContainerName: crowdstrike-falcon-init-container
              - !Ref "AWS::NoValue"
        # Conditional container definition for the falcon-init-container
        - !If
          - ConfigureFalconSensor
          -
            Name: crowdstrike-falcon-init-container
            Image: !Sub ${FalconSensorRepoParameter}
            Essential: false
            EntryPoint:
              - /bin/bash
              - -c
              - chmod u+rwx /tmp/CrowdStrike && mkdir /tmp/CrowdStrike/rootfs && cp -r /bin /etc /lib64 /usr /entrypoint-ecs.sh /tmp/CrowdStrike/rootfs && chmod -R a=rX /tmp/CrowdStrike
            MountPoints:
              - ContainerPath: /tmp/CrowdStrike
                SourceVolume: crowdstrike-falcon-volume
                ReadOnly: false
            ReadonlyRootFilesystem: true
            User: '0:0'
          - !Ref "AWS::NoValue"
      Volumes:
        !If
          - ConfigureFalconSensor
          -
            - Name: crowdstrike-falcon-volume
          - !Ref "AWS::NoValue"

  InitTableLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub ${AWS::StackName}/fargate/InitTable
      RetentionInDays: !Ref LogRetentionParameter
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  SyncTableTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family: !Sub ${AWS::StackName}-SyncTable
      Cpu: !Ref TaskCpuParameter
      Memory: !Ref TaskMemoryParameter
      EphemeralStorage:
        SizeInGiB: !Ref TaskStorageParameter
      NetworkMode: awsvpc
      RequiresCompatibilities:
        - FARGATE
      ExecutionRoleArn: !Sub ${FargateExecutionRole.Arn}
      TaskRoleArn: !Sub ${TaskRole.Arn}
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}
      ContainerDefinitions:
        - Name: !Sub ${AWS::StackName}-SyncTable
          Cpu: !Ref TaskCpuParameter
          Memory: !Ref TaskMemoryParameter
          Image:
            Fn::If:
              - CreateDatabase
              - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}/sync-table:${EnvironmentParameter}
              - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${MainCD2StackNameParameter}/sync-table:${EnvironmentParameter}
          Essential: true
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-group: !Ref SyncTableLogGroup
              awslogs-region: !Ref AWS::Region
              awslogs-stream-prefix: InitTable
          # Additional configuration for Falcon container sensor
          EntryPoint:
            !If
              - ConfigureFalconSensor
              -
                - /tmp/CrowdStrike/rootfs/lib64/ld-linux-x86-64.so.2
                - --library-path
                - /tmp/CrowdStrike/rootfs/lib64
                - /tmp/CrowdStrike/rootfs/bin/bash
                - /tmp/CrowdStrike/rootfs/entrypoint-ecs.sh
                - python
                - app.py
              - !Ref "AWS::NoValue"
          Secrets:
            !If
              - ConfigureFalconSensor
              -
                - Name: FALCONCTL_OPTS
                  ValueFrom: !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter${FalconSensorOptionsParameter}
              - !Ref "AWS::NoValue"
          LinuxParameters:
            !If
              - ConfigureFalconSensor
              - Capabilities:
                  Add:
                    - SYS_PTRACE
              - !Ref "AWS::NoValue"
          MountPoints:
            !If
              - ConfigureFalconSensor
              -
                - ContainerPath: /tmp/CrowdStrike
                  SourceVolume: crowdstrike-falcon-volume
                  ReadOnly: true
              - !Ref "AWS::NoValue"
          DependsOn:
            !If
              - ConfigureFalconSensor
              -
                - Condition: COMPLETE
                  ContainerName: crowdstrike-falcon-init-container
              - !Ref "AWS::NoValue"
        # Conditional container definition for the falcon-init-container
        - !If
          - ConfigureFalconSensor
          -
            Name: crowdstrike-falcon-init-container
            Image: !Sub ${FalconSensorRepoParameter}
            Essential: false
            EntryPoint:
              - /bin/bash
              - -c
              - chmod u+rwx /tmp/CrowdStrike && mkdir /tmp/CrowdStrike/rootfs && cp -r /bin /etc /lib64 /usr /entrypoint-ecs.sh /tmp/CrowdStrike/rootfs && chmod -R a=rX /tmp/CrowdStrike
            MountPoints:
              - ContainerPath: /tmp/CrowdStrike
                SourceVolume: crowdstrike-falcon-volume
                ReadOnly: false
            ReadonlyRootFilesystem: true
            User: '0:0'
          - !Ref "AWS::NoValue"
      Volumes:
        !If
          - ConfigureFalconSensor
          -
            - Name: crowdstrike-falcon-volume
          - !Ref "AWS::NoValue"

  SyncTableLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub ${AWS::StackName}/fargate/SyncTable
      RetentionInDays: !Ref LogRetentionParameter
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  WorkflowNotificationTopic:
    Type: AWS::SNS::Topic
    Condition: CreateNotificationTopic
    Properties:
      DisplayName: !Sub "Canvas Data 2 Synchronization Workflow (${AWS::StackName} - ${EnvironmentParameter})"
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  SlackNotificationFunction:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName: !Sub ${AWS::StackName}-${EnvironmentParameter}-slack-notification
      CodeUri: slack_notification/
      Handler: app.lambda_handler
      Runtime: python3.11
      MemorySize: 128
      Timeout: 10
      Environment:
        Variables:
          SLACK_WEBHOOK_SECRET_NAME: !Sub ${SlackWebHookURLSecretNameParameter}
          AWS_ENVIRONMENT: !Sub ${EnvironmentParameter}
          STACK_NAME: !Sub ${AWS::StackName}
      Role: !GetAtt SlackNotificationFunctionRole.Arn
      Events:
        SnsTrigger:
          Type: SNS
          Properties:
            Topic:
              Fn::If:
                - CreateNotificationTopic
                - Ref: WorkflowNotificationTopic
                - Fn::Sub: ${NotificationTopicParameter}
      Layers:
        - !Ref LambdaLayer

  SlackNotificationFunctionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action: sts:AssumeRole
            Principal:
              Service: lambda.amazonaws.com
      Policies:
        - PolicyName: SlackNotificationFunctionPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - secretsmanager:GetSecretValue
                Resource: !Sub arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${SlackWebHookURLSecretNameParameter}*
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*

  StatesExecutionRole:
    Type: "AWS::IAM::Role"
    Properties:
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Principal:
              Service:
                - !Sub states.${AWS::Region}.amazonaws.com
            Action: "sts:AssumeRole"
      Path: "/"
      Policies:
        - PolicyName: LambdaExecutionPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "lambda:InvokeFunction"
                Resource:
                  - !Sub "${ListTablesFunction.Arn}*"
        - PolicyName: ECSExecutionPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "ecs:RunTask"
                Resource:
                  - !Sub "${InitTableTaskDefinition}*"
                  - !Sub "${SyncTableTaskDefinition}*"
        - PolicyName: PassRole
          PolicyDocument:
            Statement:
            - Effect: Allow
              Action:
              - iam:PassRole
              Resource:
              - !Sub ${FargateExecutionRole.Arn}
              - !Sub ${TaskRole.Arn}
        - PolicyName: NotificationPolicy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - sns:Publish
                Resource:
                  - !If [ExistingNotificationTopic, !Ref NotificationTopicParameter, !Ref WorkflowNotificationTopic]
        - PolicyName: EventRules
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - events:PutTargets
                  - events:PutRule
                  - events:DescribeRule
                Resource:
                  - !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/StepFunctions*
        - PolicyName: logging
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*
              - Effect: Allow
                Action:
                - logs:CreateLogDelivery
                - logs:GetLogDelivery
                - logs:UpdateLogDelivery
                - logs:DeleteLogDelivery
                - logs:ListLogDeliveries
                - logs:PutResourcePolicy
                - logs:DescribeResourcePolicies
                - logs:DescribeLogGroups
                Resource: "*"
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  CD2RefreshStateMachine:
    Type: AWS::Serverless::StateMachine
    Metadata:
      cfn-lint:
        config:
          ignore_checks:
          - E3001
    Properties:
      Name: !Sub ${AWS::StackName}-state-machine
      Role: !GetAtt StatesExecutionRole.Arn
      Logging:
        Destinations:
        - CloudWatchLogsLogGroup:
            LogGroupArn: !Sub ${StateMachineLogGroup.Arn}
        IncludeExecutionData: true
        Level: ALL
      Events:
        ThreeHourlySchedule:
          Type: ScheduleV2
          Properties:
            Description: Execute the Canvas Data 2 Step Function every 3 hours
            Name: !Sub ${AWS::StackName}-CD2-Refresh-schedule
            FlexibleTimeWindow:
              Mode: FLEXIBLE
              MaximumWindowInMinutes: 10
            ScheduleExpression: rate(3 hours)
            State: ENABLED
      DefinitionSubstitutions:
        CanvasNamespaceList: !Ref NamespaceImportListParameter
      Definition:
        StartAt: StartNamespaces
        States:
          StartNamespaces:
            Type: Pass
            Next: ProcessNamespaces
            Parameters:
              namespaces.$: "States.StringSplit('${CanvasNamespaceList}', ',')"

          ProcessNamespaces:
            Type: Map
            MaxConcurrency: 0 # Run all items (namespaces) concurrently
            ItemsPath: "$.namespaces" # Iterate over the input array from StartNamespaces
            ItemSelector:
              namespace.$: "$$.Map.Item.Value"
            ItemProcessor:
              ProcessorConfig:
                Mode: INLINE
              StartAt: TableUpdates
              States:
                # The new container state that groups the data update logic
                TableUpdates:
                  Type: Parallel
                  Next: TableUpdatesComplete
                  Branches:
                    - StartAt: ListTables
                      States:
                        # Core Workflow
                        ListTables:
                          Type: Task
                          Resource: arn:aws:states:::lambda:invoke
                          OutputPath: "$.Payload"
                          Parameters:
                            Payload:
                              namespace.$: $.namespace
                              input.$: $
                            FunctionName: !Ref ListTablesFunction
                          Retry:
                            - ErrorEquals:
                              - Lambda.ServiceException
                              - Lambda.AWSLambdaException
                              - Lambda.SdkClientException
                              - Lambda.TooManyRequestsException
                              IntervalSeconds: 2
                              MaxAttempts: 6
                              BackoffRate: 6
                          Next: ProcessTables
                        ProcessTables:
                          Type: Map
                          MaxConcurrency: 10
                          ItemsPath: "$.tables"
                          Next: PivotResults
                          ItemProcessor:
                            ProcessorConfig:
                              Mode: INLINE
                            StartAt: SyncTable
                            States:
                              SyncTable:
                                Type: Task
                                Resource: arn:aws:states:::ecs:runTask.waitForTaskToken
                                Parameters:
                                  LaunchType: FARGATE
                                  Cluster: !Ref FargateCluster
                                  TaskDefinition: !Ref SyncTableTaskDefinition
                                  NetworkConfiguration:
                                    AwsvpcConfiguration:
                                      AssignPublicIp: DISABLED
                                      SecurityGroups:
                                      - !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup]
                                      Subnets:
                                      - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetA
                                      - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetB
                                  Overrides:
                                    ContainerOverrides:
                                    - Name: !Sub ${AWS::StackName}-SyncTable
                                      Environment:
                                      - Name: TASK_TOKEN
                                        Value.$: $$.Task.Token
                                      - Name: TABLE_NAME
                                        Value.$: States.JsonToString($)
                                      - Name: ENV
                                        Value: !Ref EnvironmentParameter
                                      - Name: LOG_LEVEL
                                        Value: !Ref LogLevel
                                      - Name: POWERTOOLS_METRICS_NAMESPACE
                                        Value: canvas-data-2
                                      - Name: POWERTOOLS_SERVICE_NAME
                                        Value: sync_table
                                      - Name: DB_USER_SECRET_NAME
                                        Value: !Ref DatabaseUserSecretCanvas
                                      - Name: ADMIN_SECRET_ARN
                                        Value: !If [ExistingDatabaseAdminSecret, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn]
                                      - Name: DB_CLUSTER_ARN
                                        Value: !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn]
                                      - Name: SSM_PARAMETER_NAME
                                        Value: !Sub ${SsmPathParameter}
                                      - Name: CD2_NAMESPACE
                                        Value.$: $.namespace
                                TimeoutSeconds: 43200
                                Retry:
                                  - ErrorEquals:
                                    - ECS.ServiceException
                                    - ECS.ThrottlingException
                                    - ECS.ClientException
                                    - ECS.ClusterNotFoundException
                                    - ECS.TaskSetNotFoundException
                                    - ECS.AmazonECSException
                                    IntervalSeconds: 2
                                    MaxAttempts: 6
                                    BackoffRate: 6
                                Catch:
                                  - ErrorEquals:
                                    - States.TaskFailed
                                    ResultPath: "$.error"
                                    Next: TableFailed
                                Next: CheckSyncState
                              CheckSyncState:
                                Type: Choice
                                Choices:
                                  - Variable: "$.Payload.state"
                                    StringEquals: needs_init
                                    Next: InitTable
                                Default: TableComplete
                              InitTable:
                                Type: Task
                                Resource: arn:aws:states:::ecs:runTask.waitForTaskToken
                                Parameters:
                                  LaunchType: FARGATE
                                  Cluster: !Ref FargateCluster
                                  TaskDefinition: !Ref InitTableTaskDefinition
                                  NetworkConfiguration:
                                    AwsvpcConfiguration:
                                      AssignPublicIp: DISABLED
                                      SecurityGroups:
                                      - !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup]
                                      Subnets:
                                      - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetA
                                      - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetB
                                  Overrides:
                                    ContainerOverrides:
                                    - Name: !Sub ${AWS::StackName}-InitTable
                                      Environment:
                                      - Name: TASK_TOKEN
                                        Value.$: $$.Task.Token
                                      - Name: TABLE_EVENT
                                        Value.$: States.JsonToString($.Payload)
                                      - Name: ENV
                                        Value: !Ref EnvironmentParameter
                                      - Name: LOG_LEVEL
                                        Value: !Ref LogLevel
                                      - Name: POWERTOOLS_METRICS_NAMESPACE
                                        Value: canvas-data-2
                                      - Name: POWERTOOLS_SERVICE_NAME
                                        Value: init_table
                                      - Name: DB_USER_SECRET_NAME
                                        Value: !Ref DatabaseUserSecretCanvas
                                      - Name: SSM_PARAMETER_NAME
                                        Value: !Sub ${SsmPathParameter}
                                      - Name: CD2_NAMESPACE
                                        Value.$: $.Payload.namespace
                                TimeoutSeconds: 43200
                                Retry:
                                  - ErrorEquals:
                                    - ECS.ServiceException
                                    - ECS.ThrottlingException
                                    - ECS.ClientException
                                    - ECS.ClusterNotFoundException
                                    - ECS.TaskSetNotFoundException
                                    - ECS.AmazonECSException
                                    IntervalSeconds: 2
                                    MaxAttempts: 6
                                    BackoffRate: 6
                                Catch:
                                  - ErrorEquals:
                                    - States.TaskFailed
                                    ResultPath: "$.error"
                                    Next: TableFailed
                                Next: CheckInitState
                              CheckInitState:
                                Type: Choice
                                Choices:
                                  - Variable: "$.Payload.state"
                                    StringEquals: failed
                                    Next: TableFailed
                                Default: TableComplete
                              TableFailed:
                                Type: Pass
                                OutputPath: "$.Payload"
                                End: true
                              TableComplete:
                                Type: Succeed
                                OutputPath: "$.Payload"
                        PivotResults:
                          Type: Pass
                          Next: SendNotification
                          ResultPath: null
                        SendNotification:
                          Type: Task
                          Resource: arn:aws:states:::sns:publish
                          Parameters:
                            Message.$: "$"
                            TopicArn: !If [ExistingNotificationTopic, !Ref NotificationTopicParameter, !Ref WorkflowNotificationTopic]
                          End: true
                TableUpdatesComplete:
                  Type: Succeed
            End: true

  StateMachineLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub ${AWS::StackName}/state-machine
      RetentionInDays: !Ref LogRetentionParameter
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  AthenaPostgreSQLConnector:
   Type: AWS::Serverless::Application
   Condition: CreatePrimaryCanvasAthenaConnector
   Properties:
      Location:
        ApplicationId: 'arn:aws:serverlessrepo:us-east-1:292517598671:applications/AthenaPostgreSQLConnectorWithGlueConnection'
        SemanticVersion: 2025.46.1
      Parameters:
        LambdaFunctionName: !Ref AthenaConnectorLambdaNameParameter
        CompositeHandler: PostGreSqlMuxCompositeHandler
        GlueConnection: !Ref CD2GlueConnection
        SecretName: !Sub ${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena
        DefaultScale: '0'
        LambdaRoleArn: !GetAtt AthenaPostgreSQLConnectorRole.Arn
        SecurityGroupIds: !If
          - ExistingDatabaseClientSecurityGroup
          - !Ref DatabaseClientSecurityGroupParameter
          - !Ref DatabaseClientSecurityGroup
        SpillBucket: !Ref AthenaSpillBucket
        SubnetIds: !Join [ ",", !Ref LambdaSubnetListParameter ]

  AthenaPostgreSQLCatalogConnector:
   Type: AWS::Serverless::Application
   Condition: CreateCatalogAthenaConnector
   Properties:
      Location:
        ApplicationId: 'arn:aws:serverlessrepo:us-east-1:292517598671:applications/AthenaPostgreSQLConnectorWithGlueConnection'
        SemanticVersion: 2025.46.1
      Parameters:
        LambdaFunctionName: !Sub ${AthenaConnectorLambdaNameParameter}_catalog
        CompositeHandler: PostGreSqlMuxCompositeHandler
        GlueConnection: !Ref CD2CatalogGlueConnection
        SecretName: !Sub ${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena-catalog
        DefaultScale: '0'
        LambdaRoleArn: !GetAtt AthenaPostgreSQLCatalogConnectorRole.Arn
        SecurityGroupIds: !If
          - ExistingDatabaseClientSecurityGroup
          - !Ref DatabaseClientSecurityGroupParameter
          - !Ref DatabaseClientSecurityGroup
        SpillBucket:
          Fn::ImportValue:
            Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-athena-spill-bucket
        SubnetIds: !Join [ ",", !Ref LambdaSubnetListParameter ]

  CD2GlueConnection:
    Type: AWS::Glue::Connection
    Condition: CreatePrimaryCanvasAthenaConnector
    Properties:
      CatalogId: !Ref AWS::AccountId
      ConnectionInput:
        Name: !Sub ${AWS::StackName}-cd2
        Description: "Connection to cd2 PostgreSQL"
        ConnectionType: POSTGRESQL
        PhysicalConnectionRequirements:
          SubnetId: !Join [ ",", !Ref LambdaSubnetListParameter ]
          SecurityGroupIdList:
            - !If
              - ExistingDatabaseClientSecurityGroup
              - !Ref DatabaseClientSecurityGroupParameter
              - !Ref DatabaseClientSecurityGroup
        ConnectionProperties:
          HOST: !Sub ${AuroraDatabaseCluster.Endpoint.Address}
          DATABASE: "cd2"
          JDBC_PARAMS: "sslmode=verify-ca&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory&stringtype=unspecified"
          PORT: !Sub ${AuroraDatabaseCluster.Endpoint.Port}
        AthenaProperties:
          spill_prefix: "cd2"
          default_scale: "0"
          disable_spill_encryption: "false"
          spill_bucket: !Ref AthenaSpillBucket
        AuthenticationConfiguration:
          AuthenticationType: "BASIC"
          SecretArn: !Ref DatabaseUserSecretAthena

  CD2CatalogGlueConnection:
    Type: AWS::Glue::Connection
    Condition: CreateCatalogAthenaConnector
    Properties:
      CatalogId: !Ref AWS::AccountId
      ConnectionInput:
        Name: !Sub ${AWS::StackName}-cd2-catalog
        Description: "Connection to cd2-catalog PostgreSQL"
        ConnectionType: POSTGRESQL
        PhysicalConnectionRequirements:
          SubnetId: !Join [ ",", !Ref LambdaSubnetListParameter ]
          SecurityGroupIdList:
            - !If
              - ExistingDatabaseClientSecurityGroup
              - !Ref DatabaseClientSecurityGroupParameter
              - !Ref DatabaseClientSecurityGroup
        ConnectionProperties:
          HOST:
            Fn::ImportValue:
              Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-aurora-cluster-endpoint-address
          DATABASE: "cd2_catalog"
          JDBC_PARAMS: "sslmode=verify-ca&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory&stringtype=unspecified"
          PORT:
            Fn::ImportValue:
              Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-aurora-cluster-endpoint-port
        AthenaProperties:
          spill_prefix: "cd2_catalog"
          default_scale: "0"
          disable_spill_encryption: "false"
          spill_bucket: !Sub ${ResourcePrefixParameter}-athena-spill-${AWS::AccountId}-${AWS::Region}
        AuthenticationConfiguration:
          AuthenticationType: "BASIC"
          SecretArn: !Ref DatabaseUserSecretAthenaCatalog

  AthenaSpillBucket:
    Type: AWS::S3::Bucket
    Condition: CreatePrimaryCanvasAthenaConnector
    Properties:
      BucketName: !Sub ${ResourcePrefixParameter}-athena-spill-${AWS::AccountId}-${AWS::Region}
      BucketEncryption:
        ServerSideEncryptionConfiguration:
        - ServerSideEncryptionByDefault:
            SSEAlgorithm: aws:kms
            KMSMasterKeyID:
              Fn::ImportValue: !Sub ${ResourcePrefixParameter}-lakehouse--keyArn
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  AthenaPostgreSQLConnectorRole:
    Type: AWS::IAM::Role
    Condition: CreatePrimaryCanvasAthenaConnector
    Properties:
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  AthenaPostgreSQLConnectorExecutionPolicy:
    Type: AWS::IAM::Policy
    Condition: CreatePrimaryCanvasAthenaConnector
    Properties:
      Roles:
      - !Ref AthenaPostgreSQLConnectorRole
      PolicyName: FunctionExecutionPolicy
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Action:
          - secretsmanager:GetSecretValue
          Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena*
        - Effect: Allow
          Action:
          - logs:CreateLogGroup
          Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*
        - Effect: Allow
          Action:
          - logs:CreateLogStream
          - logs:PutLogEvents
          Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/athenafederatedcatalog*
        - Effect: Allow
          Action:
            - athena:GetQueryExecution
            - s3:ListAllMyBuckets
          Resource: '*'
        - Effect: Allow
          Action:
          - ec2:CreateNetworkInterface
          - ec2:DeleteNetworkInterface
          - ec2:DescribeNetworkInterfaces
          - ec2:DetachNetworkInterface
          Resource: '*'
        - Effect: Allow
          Action:
          - s3:GetObject
          - s3:ListBucket
          - s3:GetBucketLocation
          - s3:GetObjectVersion
          - s3:PutObject
          - s3:PutObjectAcl
          - s3:GetLifecycleConfiguration
          - s3:PutLifecycleConfiguration
          - s3:DeleteObject
          Resource:
            - !Sub ${AthenaSpillBucket.Arn}
            - !Sub ${AthenaSpillBucket.Arn}/*
        - Effect: Allow
          Action:
          - kms:Decrypt
          Resource: !GetAtt SecretsKmsKey.Arn
        - Effect: Allow
          Action:
          - kms:GenerateDataKey
          Resource:
            Fn::ImportValue: !Sub ${ResourcePrefixParameter}-lakehouse--keyArn
        - Action:
          - glue:GetConnection
          Effect: Allow
          Resource:
            - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:connection/${CD2GlueConnection}'
            - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog'

  AthenaPostgreSQLCatalogConnectorRole:
    Type: AWS::IAM::Role
    Condition: CreateCatalogAthenaConnector
    Properties:
      ManagedPolicyArns:
      - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  AthenaPostgreSQLCatalogConnectorExecutionPolicy:
    Type: AWS::IAM::Policy
    Condition: CreateCatalogAthenaConnector
    Properties:
      Roles:
      - !Ref AthenaPostgreSQLCatalogConnectorRole
      PolicyName: FunctionExecutionPolicy
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Action:
          - secretsmanager:GetSecretValue
          Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena-catalog*
        - Effect: Allow
          Action:
          - logs:CreateLogGroup
          Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*
        - Effect: Allow
          Action:
          - logs:CreateLogStream
          - logs:PutLogEvents
          Resource: !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/athenafederatedcatalog*
        - Effect: Allow
          Action:
            - athena:GetQueryExecution
            - s3:ListAllMyBuckets
          Resource: '*'
        - Effect: Allow
          Action:
          - ec2:CreateNetworkInterface
          - ec2:DeleteNetworkInterface
          - ec2:DescribeNetworkInterfaces
          - ec2:DetachNetworkInterface
          Resource: '*'
        - Effect: Allow
          Action:
          - s3:GetObject
          - s3:ListBucket
          - s3:GetBucketLocation
          - s3:GetObjectVersion
          - s3:PutObject
          - s3:PutObjectAcl
          - s3:GetLifecycleConfiguration
          - s3:PutLifecycleConfiguration
          - s3:DeleteObject
          Resource:
            - Fn::ImportValue:
                Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-athena-spill-bucket
            - Fn::Sub:
              - ${AthenaS3SpillBucketArn}/*
              - AthenaS3SpillBucketArn:
                  Fn::ImportValue:
                    Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-athena-spill-bucket
        - Effect: Allow
          Action:
          - kms:Decrypt
          Resource:
            - Fn::ImportValue:
                Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-secrets-kms-key-arn
        - Effect: Allow
          Action:
          - kms:GenerateDataKey
          Resource:
            Fn::ImportValue: !Sub ${ResourcePrefixParameter}-lakehouse--keyArn
        - Action:
          - glue:GetConnection
          Effect: Allow
          Resource:
            - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:connection/${CD2CatalogGlueConnection}'
            - !Sub 'arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog'

  # CanvasDataCatalog:
  #   Type: AWS::Athena::DataCatalog
  #   Condition: CreateAthenaConnector
  #   Properties:
  #     Name: canvas
  #     ConnectionType: POSTGRESQL
  #     Description: PostgreSQL connector to cd2
  #     Type: FEDERATED
  #     Parameters:
  #       connection-arn: !Sub arn:aws:glue:${AWS::Region}:${AWS::AccountId}:connection/${CD2GlueConnection}
  #       lambda-role-arn: !GetAtt AthenaPostgreSQLConnectorRole.Arn
  #     Tags:
  #       - Key: !Sub ${TagNameParameter}
  #         Value: !Sub ${TagValueParameter}

  CanvasDataCatalog:
    Type: AWS::Athena::DataCatalog
    Condition: CreateAthenaConnector
    Properties:
      Name: !Sub ${DatabaseCd2UserParameter}
      ConnectionType: POSTGRESQL
      Description: PostgreSQL connector to cd2
      Type: LAMBDA
      Parameters:
        function: !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${AthenaConnectorLambdaNameParameter}
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  CatalogDataCatalog:
    Type: AWS::Athena::DataCatalog
    Condition: CreateCatalogAthenaConnector
    Properties:
      Name: !Sub ${DatabaseCd2UserParameter}-catalog
      ConnectionType: POSTGRESQL
      Description: PostgreSQL connector to cd2 catalog
      Type: LAMBDA
      Parameters:
        function: !Sub arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${AthenaConnectorLambdaNameParameter}_catalog
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  # Secret for PostgreSQL Athena connector "athena" user
  DatabaseUserSecretAthena:
    Type: AWS::SecretsManager::Secret
    Condition: CreatePrimaryCanvasAthenaConnector
    Properties:
      Name: !Sub ${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena
      Description: Database user for Athena PostgreSQL connector
      GenerateSecretString:
        SecretStringTemplate: !Sub '{"username": "athena"}'
        GenerateStringKey: password
        PasswordLength: 128
        ExcludePunctuation: true
      KmsKeyId: !Ref SecretsKmsKey
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  # Attach Athena database user secret to the cluster
  DatabaseUserSecretAttachmentAthena:
    Type: AWS::SecretsManager::SecretTargetAttachment
    Condition: CreatePrimaryCanvasAthenaConnector
    Properties:
      SecretId: !Ref DatabaseUserSecretAthena
      TargetId: !Ref AuroraDatabaseCluster
      TargetType: AWS::RDS::DBCluster

  DatabaseUserSecretAthenaCatalog:
    Type: AWS::SecretsManager::Secret
    Condition: CreateCatalogAthenaConnector
    Properties:
      Name: !Sub ${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena-catalog
      Description: Database user for Athena PostgreSQL connector for the catalog instance
      GenerateSecretString:
        SecretStringTemplate: !Sub '{"username": "athena_catalog"}'
        GenerateStringKey: password
        PasswordLength: 128
        ExcludePunctuation: true
      KmsKeyId:
        Fn::ImportValue:
          Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-secrets-kms-key-id
      Tags:
        - Key: !Sub ${TagNameParameter}
          Value: !Sub ${TagValueParameter}

  # Attach Athena database user secret for the canvas catalog instance to the cluster
  DatabaseUserSecretAttachmentAthenaCatalog:
    Type: AWS::SecretsManager::SecretTargetAttachment
    Condition: CreateCatalogAthenaConnector
    Properties:
      SecretId: !Ref DatabaseUserSecretAthenaCatalog
      TargetId:
        Fn::ImportValue:
          Fn::Sub: ${ResourcePrefixParameter}-canvas-data-2-${EnvironmentParameter}-aurora-cluster-identifier
      TargetType: AWS::RDS::DBCluster

  LambdaLayer:
    Type: AWS::Serverless::LayerVersion
    Properties:
      LayerName: !Sub ${AWS::StackName}
      Description: Additional Python Libraries for CD2 Lambda functions.
      ContentUri: lambda-layers
      CompatibleRuntimes:
        - python3.12

Outputs:
  SecretsKmsKeyArn:
    Value: !GetAtt SecretsKmsKey.Arn
    Description: Canvas Data 2 Secrets KMS key ARN
    Condition: CreateDatabase
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-secrets-kms-key-arn
  SecretsKmsKeyId:
    Value: !Ref SecretsKmsKey
    Description: Canvas Data 2 Secrets KMS key ID
    Condition: CreateDatabase
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-secrets-kms-key-id
  DatabaseAdminSecretArn:
    Value: !If [ExistingDatabaseAdminSecret, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn]
    Description: Canvas Data 2 database admin user secret ARN
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-admin-secret-arn
  DatabaseUserSecretArn:
    Value: !Ref DatabaseUserSecretCanvas
    Description: Canvas Data 2 database user secret ARN
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-user-secret-arn
  DatabaseUserSecretAthenaArn:
    Value: !Ref DatabaseUserSecretAthena
    Description: Canvas Data 2 database Athena secret ARN
    Condition: CreateDatabase
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-athena-secret-arn
  AuroraClusterArn:
    Value: !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn]
    Description: Canvas Data 2 Aurora cluster ARN
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-aurora-cluster-arn
  AuroraClusterDBClusterIdentifier:
    Value: !Ref AuroraDatabaseCluster
    Description: Canvas Data 2 Aurora cluster DB Cluster Identifier
    Condition: CreateDatabase
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-aurora-cluster-identifier
  AuroraClusterEndpoint:
    Value: !GetAtt AuroraDatabaseCluster.Endpoint.Address
    Description: Canvas Data 2 Aurora cluster ARN
    Condition: CreateDatabase
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-aurora-cluster-endpoint-address
  AuroraClusterPort:
    Value: !GetAtt AuroraDatabaseCluster.Endpoint.Port
    Description: Canvas Data 2 Aurora cluster ARN
    Condition: CreateDatabase
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-aurora-cluster-endpoint-port
  AthenaSpillBucketArn:
    Value: !If [ExistingDatabase, '', !GetAtt AthenaSpillBucket.Arn]
    Description: Canvas Data 2 Athena S3 Spill Bucket ARN
    Condition: CreatePrimaryCanvasAthenaConnector
    Export:
      Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-athena-spill-bucket