diff --git a/aws_emr_blog_v3/cloudformation/ranger-server.template b/aws_emr_blog_v3/cloudformation/ranger-server.template index 80f9f26..a772e0e 100644 --- a/aws_emr_blog_v3/cloudformation/ranger-server.template +++ b/aws_emr_blog_v3/cloudformation/ranger-server.template @@ -1 +1,512 @@ -Description: CloudFormation template to create Apache Ranger server with Solr Parameters: S3Bucket: Description: S3Bucket where artifacts are stored Type: String Default: aws-bigdata-blog S3Key: Description: S3Key of the Lambda code Type: String VPC: Description: VPC ID for creating the EMR cluster Type: AWS::EC2::VPC::Id Subnet: Description: Subnet ID for creating the EMR cluster Type: AWS::EC2::Subnet::Id ProjectVersion: Default: 3.0 Description: Project version Type: String AllowedValues: - 3.0 KeyPairName: Description: Name of an existing EC2 KeyPair to enable SSH to the instances Type: AWS::EC2::KeyPair::KeyName DBHostName: Description: HostName of the database Type: String DBRootPassword: Description: Root password of database Type: String ADAdminUsername: Description: Admin username of AD Type: String ADAdminPassword: Description: Admin password for AD Type: String InstanceType: Description: Instance type of the Ranger Server Type: String Default: r5.2xlarge AllowedValues: - t1.micro - t2.nano - t2.micro - t2.small - t2.medium - t2.large - m1.small - m1.medium - m1.large - m1.xlarge - m2.xlarge - m2.2xlarge - m2.4xlarge - m3.medium - m3.large - m3.xlarge - m3.2xlarge - m4.large - m4.xlarge - m4.2xlarge - m4.4xlarge - m4.10xlarge - c1.medium - c1.xlarge - c3.large - c3.xlarge - c3.2xlarge - c3.4xlarge - c3.8xlarge - c4.large - c4.xlarge - c4.2xlarge - c4.4xlarge - c4.8xlarge - g2.2xlarge - g2.8xlarge - r3.large - r3.xlarge - r3.2xlarge - r3.4xlarge - r3.8xlarge - i2.xlarge - i2.2xlarge - i2.4xlarge - i2.8xlarge - d2.xlarge - d2.2xlarge - d2.4xlarge - d2.8xlarge - hi1.4xlarge - hs1.8xlarge - cr1.8xlarge - cc2.8xlarge - cg1.4xlarge - r5.4xlarge - r5.2xlarge - r5.xlarge ConstraintDescription: must be a valid EC2 instance type. LDAPHostPrivateIP: Description: IP Address of the SimpleAD server Type: String DomainDNSName: AllowedPattern: '[a-zA-Z0-9\-]+\..+' Default: awsemr.com Description: The Active Directory domain that you want to establish the cross-realm trust with e.g., awsemr.com MaxLength: '25' MinLength: '3' Type: String LDAPSearchBase: Description: Base DN SimpleAD server Type: String Default: dc=awsemr,dc=com LDAPBindUserName: Description: BindUser SimpleAD server Type: String Default: binduser AllowedValues: - binduser LDAPBindPassword: Description: BindPassword SimpleAD server Type: String Default: Bind@User123 rangerVersion: Description: RangerVersion Type: String Default: '2.0' AllowedValues: - '2.0' AttachAdditionalSourcePrefixToSG: Description: Attaches additional sources to EMR Master SG Default: false Type: String AllowedValues: [true, false] CIDRAccessToPrivateSubnetResources: Description: IP address range (in CIDR notation) of the client that will be allowed to connect to the cluster using SSH e.g., 203.0.113.5/32 AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) Type: String MinLength: '9' MaxLength: '18' Default: 10.0.0.0/16 ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x AdditionalSourcePrefixToSG: Description: Sources that are allowd to access the Ranger Instance. Should be a source prefix e.g., pl-xxx Type: String Mappings: AWSInstanceType2Arch: t1.micro: Arch: PV64 t2.nano: Arch: HVM64 t2.micro: Arch: HVM64 t2.small: Arch: HVM64 t2.medium: Arch: HVM64 t2.large: Arch: HVM64 m3.medium: Arch: HVM64 m3.large: Arch: HVM64 m3.xlarge: Arch: HVM64 m3.2xlarge: Arch: HVM64 m4.large: Arch: HVM64 m4.xlarge: Arch: HVM64 m4.2xlarge: Arch: HVM64 m4.4xlarge: Arch: HVM64 m4.10xlarge: Arch: HVM64 c1.medium: Arch: PV64 c1.xlarge: Arch: PV64 c3.large: Arch: HVM64 c3.xlarge: Arch: HVM64 c3.2xlarge: Arch: HVM64 c3.4xlarge: Arch: HVM64 c3.8xlarge: Arch: HVM64 c4.large: Arch: HVM64 c4.xlarge: Arch: HVM64 c4.2xlarge: Arch: HVM64 c4.4xlarge: Arch: HVM64 c4.8xlarge: Arch: HVM64 r3.large: Arch: HVM64 r3.xlarge: Arch: HVM64 r3.2xlarge: Arch: HVM64 r3.4xlarge: Arch: HVM64 r3.8xlarge: Arch: HVM64 i2.xlarge: Arch: HVM64 i2.2xlarge: Arch: HVM64 i2.4xlarge: Arch: HVM64 i2.8xlarge: Arch: HVM64 d2.xlarge: Arch: HVM64 d2.2xlarge: Arch: HVM64 d2.4xlarge: Arch: HVM64 d2.8xlarge: Arch: HVM64 hi1.4xlarge: Arch: HVM64 hs1.8xlarge: Arch: HVM64 cr1.8xlarge: Arch: HVM64 cc2.8xlarge: Arch: HVM64 r5.xlarge: Arch: HVM64 r5.2xlarge: Arch: HVM64 r5.4xlarge: Arch: HVM64 AWSRegionArch2AMI: us-east-1: PV64: ami-0023040df18933030 HVM64: ami-0915e09cc7ceee3ab us-east-2: PV64: ami-0ea2bc03f34d1ada4 HVM64: ami-097834fcb3081f51a us-west-2: PV64: ami-00cb981adfcebb519 HVM64: ami-01f08ef3e76b957e5 us-west-1: PV64: ami-0027eed75be6f3bf4 HVM64: ami-014743cb7690ea737 eu-west-1: PV64: ami-00b6370b096f24de2 HVM64: ami-00890f614e48ce866 eu-central-1: PV64: ami-0001160eb97d88825 HVM64: ami-03ab4e8f1d88ce614 ap-northeast-1: PV64: ami-0095079896fca4cca HVM64: ami-0318ecd6d05daa212 ap-northeast-2: PV64: NOT_SUPPORTED HVM64: ami-09391a0ad9f9243b6 ap-southeast-1: PV64: ami-021f73ba029345fb1 HVM64: ami-0dff4318d85149d5d ap-southeast-2: PV64: ami-01c6bf0aeb3c63052 HVM64: ami-050e1ec030abb8dde sa-east-1: PV64: ami-0015527da78932f76 HVM64: ami-03e1e4abf50e14ded cn-north-1: PV64: ami-7f84361b HVM64: ami-021321e9bc16d5186 Conditions: AttachAdditionalSourcePrefixToSG: !Equals [true, !Ref AttachAdditionalSourcePrefixToSG] Resources: ManagedInstanceRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: Service: - ssm.amazonaws.com - ec2.amazonaws.com Action: sts:AssumeRole ManagedPolicyArns: - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM - arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess Path: "/" AllowSecretsRetrievalPolivy: Type: 'AWS::IAM::Policy' Properties: PolicyName: AllowSecretsRetrieval PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - secretsmanager:GetSecretValue - secretsmanager:ListSecrets - secretsmanager:DescribeSecret Resource: - !Join ['', ['arn:aws:secretsmanager:', !Ref "AWS::Region", ':', !Ref "AWS::AccountId", ':secret:emr/ranger*']] Roles: - !Ref ManagedInstanceRole ManagedInstanceProfile: Type: AWS::IAM::InstanceProfile Properties: Path: "/" Roles: - !Ref ManagedInstanceRole sgRangerAdminServer: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: Open Up all Ranger Server ports VpcId: !Ref 'VPC' SecurityGroupIngress: - IpProtocol: tcp FromPort: '6080' ToPort: '6080' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '6182' ToPort: '6182' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '8983' ToPort: '8983' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '8984' ToPort: '8984' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '22' ToPort: '22' CidrIp: !Ref CIDRAccessToPrivateSubnetResources sgRangerAdminServerWithAdditions: Type: AWS::EC2::SecurityGroup Condition: AttachAdditionalSourcePrefixToSG Properties: GroupDescription: Open Up all Ranger Server ports VpcId: !Ref 'VPC' SecurityGroupIngress: - IpProtocol: tcp FromPort: '6080' ToPort: '6080' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '6080' ToPort: '6080' SourcePrefixListId: !Ref AdditionalSourcePrefixToSG - IpProtocol: tcp FromPort: '6182' ToPort: '6182' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '6182' ToPort: '6182' SourcePrefixListId: !Ref AdditionalSourcePrefixToSG - IpProtocol: tcp FromPort: '8983' ToPort: '8983' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '8983' ToPort: '8983' SourcePrefixListId: !Ref AdditionalSourcePrefixToSG - IpProtocol: tcp FromPort: '8984' ToPort: '8984' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '8984' ToPort: '8984' SourcePrefixListId: !Ref AdditionalSourcePrefixToSG - IpProtocol: tcp FromPort: '22' ToPort: '22' CidrIp: !Ref CIDRAccessToPrivateSubnetResources - IpProtocol: tcp FromPort: '22' ToPort: '22' SourcePrefixListId: !Ref AdditionalSourcePrefixToSG RangerLogGroup: Type: "AWS::Logs::LogGroup" Properties: RetentionInDays: 14 #LogGroupName: rangerlogs-${AWS::StackName} myEC2: Type: AWS::EC2::Instance Metadata: AWS::CloudFormation::Init: configSets: ascending: - CloudwatchLogs - RangerServer CloudwatchLogs: files: '/etc/awslogs/awslogs.conf': content: !Sub | [general] # Path to the CloudWatch Logs agent's state file. The agent uses this file to maintain # client side state across its executions. state_file = /var/lib/awslogs/agent-state [ranger_admin_setup_log] datetime_format = %Y-%m-%d %H:%M:%S,%f file = /tmp/create-ranger-server-output.log buffer_duration = 500 log_stream_name = adminsetuplog-{instance_id} initial_position = start_of_file log_group_name = ${RangerLogGroup} [ranger_admin_log] datetime_format = %Y-%m-%d %H:%M:%S,%f file = /usr/lib/ranger/logs/admin/logs/ranger-admin* buffer_duration = 500 log_stream_name = adminlog-{instance_id} initial_position = start_of_file log_group_name = ${RangerLogGroup} commands: startawslogs: command: !Sub | sudo yum update -y sudo yum install -y awslogs sudo service awslogs restart sudo chkconfig awslogs on RangerServer: commands: installrangerserver: command: !Join ['', ['bash /tmp/install-ranger-admin-server.sh ', !Ref 'LDAPHostPrivateIP', ' ', !Ref 'LDAPSearchBase', ' ', !Ref 'LDAPBindUserName', '@',!Ref 'DomainDNSName' , ' ', !Ref 'LDAPBindPassword', ' ', !Ref 'rangerVersion', ' ', !Join ['', ['s3://', !Ref S3Bucket, '/', !Ref S3Key]], ' ', !Ref 'ProjectVersion', ' ', !Ref 'DBHostName', ' ', !Ref 'DBRootPassword', ' '," >\ \ create-ranger-server-output.log \n"]] Properties: SubnetId: !Ref 'Subnet' IamInstanceProfile: !Ref ManagedInstanceProfile SecurityGroupIds: - !If [ AttachAdditionalSourcePrefixToSG, !Ref sgRangerAdminServerWithAdditions, !Ref sgRangerAdminServer ] ImageId: !FindInMap [AWSRegionArch2AMI, !Ref 'AWS::Region', !FindInMap [AWSInstanceType2Arch, !Ref 'InstanceType', Arch]] InstanceType: !Ref 'InstanceType' KeyName: !Ref 'KeyPairName' BlockDeviceMappings: - DeviceName: /dev/xvda Ebs: VolumeType: gp2 VolumeSize: '100' DeleteOnTermination: 'true' UserData: !Base64 Fn::Join: - '' - - '#!/bin/bash ' - 'cd /tmp ' - 'aws s3 cp ' - !Join ['', ['s3://', !Ref S3Bucket, '/', !Ref S3Key, '/', !Ref ProjectVersion]] - '/scripts/install-ranger-admin-server.sh . ' - 'yum update aws-cfn-bootstrap ' - '# Install the files and packages from the metadata ' - '/opt/aws/bin/cfn-init ' - ' --stack ' - !Ref 'AWS::StackName' - ' --resource myEC2 ' - ' --configsets ascending ' - ' --region ' - !Ref 'AWS::Region' - ' ' Tags: - Key: Name Value: RangerServer Outputs: IPAddress: Description: IP address of the Ranger server Value: !GetAtt [myEC2, PrivateIp] PrivateDNS: Description: IP address of the Ranger server Value: !GetAtt [myEC2, PrivateDnsName] \ No newline at end of file +Description: CloudFormation template to create Apache Ranger server with Solr +Parameters: + S3Bucket: + Description: S3Bucket where artifacts are stored + Type: String + Default: aws-bigdata-blog + S3Key: + Description: S3Key of the Lambda code + Type: String + VPC: + Description: VPC ID for creating the EMR cluster + Type: AWS::EC2::VPC::Id + Subnet: + Description: Subnet ID for creating the EMR cluster + Type: AWS::EC2::Subnet::Id + ProjectVersion: + Default: 3.0 + Description: Project version + Type: String + AllowedValues: + - 3.0 + KeyPairName: + Description: Name of an existing EC2 KeyPair to enable SSH to the instances + Type: AWS::EC2::KeyPair::KeyName + DBHostName: + Description: HostName of the database + Type: String + DBRootPassword: + Description: Root password of database + Type: String + ADAdminUsername: + Description: Admin username of AD + Type: String + ADAdminPassword: + Description: Admin password for AD + Type: String + InstanceType: + Description: Instance type of the Ranger Server + Type: String + Default: r5.2xlarge + AllowedValues: + - t1.micro + - t2.nano + - t2.micro + - t2.small + - t2.medium + - t2.large + - m1.small + - m1.medium + - m1.large + - m1.xlarge + - m2.xlarge + - m2.2xlarge + - m2.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - c1.medium + - c1.xlarge + - c3.large + - c3.xlarge + - c3.2xlarge + - c3.4xlarge + - c3.8xlarge + - c4.large + - c4.xlarge + - c4.2xlarge + - c4.4xlarge + - c4.8xlarge + - g2.2xlarge + - g2.8xlarge + - r3.large + - r3.xlarge + - r3.2xlarge + - r3.4xlarge + - r3.8xlarge + - i2.xlarge + - i2.2xlarge + - i2.4xlarge + - i2.8xlarge + - d2.xlarge + - d2.2xlarge + - d2.4xlarge + - d2.8xlarge + - hi1.4xlarge + - hs1.8xlarge + - cr1.8xlarge + - cc2.8xlarge + - cg1.4xlarge + - r5.4xlarge + - r5.2xlarge + - r5.xlarge + ConstraintDescription: must be a valid EC2 instance type. + LDAPHostPrivateIP: + Description: IP Address of the SimpleAD server + Type: String + DomainDNSName: + AllowedPattern: '[a-zA-Z0-9\-]+\..+' + Default: awsemr.com + Description: The Active Directory domain that you want to establish the cross-realm + trust with e.g., awsemr.com + MaxLength: '25' + MinLength: '3' + Type: String + LDAPSearchBase: + Description: Base DN SimpleAD server + Type: String + Default: dc=awsemr,dc=com + LDAPBindUserName: + Description: BindUser SimpleAD server + Type: String + Default: binduser + AllowedValues: + - binduser + LDAPBindPassword: + Description: BindPassword SimpleAD server + Type: String + Default: Bind@User123 + rangerVersion: + Description: RangerVersion + Type: String + Default: '2.0' + AllowedValues: + - '2.0' + AttachAdditionalSourcePrefixToSG: + Description: Attaches additional sources to EMR Master SG + Default: false + Type: String + AllowedValues: [true, false] + CIDRAccessToPrivateSubnetResources: + Description: IP address range (in CIDR notation) of the client that will be allowed + to connect to the cluster using SSH e.g., 203.0.113.5/32 + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + Type: String + MinLength: '9' + MaxLength: '18' + Default: 10.0.0.0/16 + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x + AdditionalSourcePrefixToSG: + Description: Sources that are allowd to access the Ranger Instance. Should be a source prefix + e.g., pl-xxx + Type: String +Mappings: + AWSInstanceType2Arch: + t1.micro: + Arch: PV64 + t2.nano: + Arch: HVM64 + t2.micro: + Arch: HVM64 + t2.small: + Arch: HVM64 + t2.medium: + Arch: HVM64 + t2.large: + Arch: HVM64 + m3.medium: + Arch: HVM64 + m3.large: + Arch: HVM64 + m3.xlarge: + Arch: HVM64 + m3.2xlarge: + Arch: HVM64 + m4.large: + Arch: HVM64 + m4.xlarge: + Arch: HVM64 + m4.2xlarge: + Arch: HVM64 + m4.4xlarge: + Arch: HVM64 + m4.10xlarge: + Arch: HVM64 + c1.medium: + Arch: PV64 + c1.xlarge: + Arch: PV64 + c3.large: + Arch: HVM64 + c3.xlarge: + Arch: HVM64 + c3.2xlarge: + Arch: HVM64 + c3.4xlarge: + Arch: HVM64 + c3.8xlarge: + Arch: HVM64 + c4.large: + Arch: HVM64 + c4.xlarge: + Arch: HVM64 + c4.2xlarge: + Arch: HVM64 + c4.4xlarge: + Arch: HVM64 + c4.8xlarge: + Arch: HVM64 + r3.large: + Arch: HVM64 + r3.xlarge: + Arch: HVM64 + r3.2xlarge: + Arch: HVM64 + r3.4xlarge: + Arch: HVM64 + r3.8xlarge: + Arch: HVM64 + i2.xlarge: + Arch: HVM64 + i2.2xlarge: + Arch: HVM64 + i2.4xlarge: + Arch: HVM64 + i2.8xlarge: + Arch: HVM64 + d2.xlarge: + Arch: HVM64 + d2.2xlarge: + Arch: HVM64 + d2.4xlarge: + Arch: HVM64 + d2.8xlarge: + Arch: HVM64 + hi1.4xlarge: + Arch: HVM64 + hs1.8xlarge: + Arch: HVM64 + cr1.8xlarge: + Arch: HVM64 + cc2.8xlarge: + Arch: HVM64 + r5.xlarge: + Arch: HVM64 + r5.2xlarge: + Arch: HVM64 + r5.4xlarge: + Arch: HVM64 + AWSRegionArch2AMI: + us-east-1: + PV64: ami-0023040df18933030 + HVM64: ami-0915e09cc7ceee3ab + us-east-2: + PV64: ami-0ea2bc03f34d1ada4 + HVM64: ami-097834fcb3081f51a + us-west-2: + PV64: ami-00cb981adfcebb519 + HVM64: ami-01f08ef3e76b957e5 + us-west-1: + PV64: ami-0027eed75be6f3bf4 + HVM64: ami-014743cb7690ea737 + eu-west-1: + PV64: ami-00b6370b096f24de2 + HVM64: ami-00890f614e48ce866 + eu-central-1: + PV64: ami-0001160eb97d88825 + HVM64: ami-03ab4e8f1d88ce614 + ap-northeast-1: + PV64: ami-0095079896fca4cca + HVM64: ami-0318ecd6d05daa212 + ap-northeast-2: + PV64: NOT_SUPPORTED + HVM64: ami-09391a0ad9f9243b6 + ap-southeast-1: + PV64: ami-021f73ba029345fb1 + HVM64: ami-0dff4318d85149d5d + ap-southeast-2: + PV64: ami-01c6bf0aeb3c63052 + HVM64: ami-050e1ec030abb8dde + sa-east-1: + PV64: ami-0015527da78932f76 + HVM64: ami-03e1e4abf50e14ded + cn-north-1: + PV64: ami-7f84361b + HVM64: ami-021321e9bc16d5186 + +Conditions: + AttachAdditionalSourcePrefixToSG: !Equals [true, !Ref AttachAdditionalSourcePrefixToSG] + +Resources: + ManagedInstanceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ssm.amazonaws.com + - ec2.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM + - arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess + - arn:aws:iam::aws:policy/CloudWatchLogsFullAccess + Path: "/" + AllowSecretsRetrievalPolivy: + Type: 'AWS::IAM::Policy' + Properties: + PolicyName: AllowSecretsRetrieval + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + - secretsmanager:ListSecrets + - secretsmanager:DescribeSecret + Resource: + - !Join ['', ['arn:aws:secretsmanager:', !Ref "AWS::Region", ':', !Ref "AWS::AccountId", ':secret:emr/ranger*']] + Roles: + - !Ref ManagedInstanceRole + ManagedInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: "/" + Roles: + - !Ref ManagedInstanceRole + sgRangerAdminServer: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Open Up all Ranger Server ports + VpcId: !Ref 'VPC' + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: '6080' + ToPort: '6080' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '6182' + ToPort: '6182' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '8983' + ToPort: '8983' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '8984' + ToPort: '8984' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '22' + ToPort: '22' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + sgRangerAdminServerWithAdditions: + Type: AWS::EC2::SecurityGroup + Condition: AttachAdditionalSourcePrefixToSG + Properties: + GroupDescription: Open Up all Ranger Server ports + VpcId: !Ref 'VPC' + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: '6080' + ToPort: '6080' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '6080' + ToPort: '6080' + SourcePrefixListId: !Ref AdditionalSourcePrefixToSG + - IpProtocol: tcp + FromPort: '6182' + ToPort: '6182' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '6182' + ToPort: '6182' + SourcePrefixListId: !Ref AdditionalSourcePrefixToSG + - IpProtocol: tcp + FromPort: '8983' + ToPort: '8983' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '8983' + ToPort: '8983' + SourcePrefixListId: !Ref AdditionalSourcePrefixToSG + - IpProtocol: tcp + FromPort: '8984' + ToPort: '8984' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '8984' + ToPort: '8984' + SourcePrefixListId: !Ref AdditionalSourcePrefixToSG + - IpProtocol: tcp + FromPort: '22' + ToPort: '22' + CidrIp: !Ref CIDRAccessToPrivateSubnetResources + - IpProtocol: tcp + FromPort: '22' + ToPort: '22' + SourcePrefixListId: !Ref AdditionalSourcePrefixToSG + RangerLogGroup: + Type: "AWS::Logs::LogGroup" + Properties: + RetentionInDays: 14 + #LogGroupName: rangerlogs-${AWS::StackName} + myEC2: + Type: AWS::EC2::Instance + Metadata: + AWS::CloudFormation::Init: + configSets: + ascending: + - CloudwatchLogs + - RangerServer + CloudwatchLogs: + files: + '/etc/awslogs/awslogs.conf': + content: !Sub | + [general] + # Path to the CloudWatch Logs agent's state file. The agent uses this file to maintain + # client side state across its executions. + state_file = /var/lib/awslogs/agent-state + [ranger_admin_setup_log] + datetime_format = %Y-%m-%d %H:%M:%S,%f + file = /tmp/create-ranger-server-output.log + buffer_duration = 500 + log_stream_name = adminsetuplog-{instance_id} + initial_position = start_of_file + log_group_name = ${RangerLogGroup} + [ranger_admin_log] + datetime_format = %Y-%m-%d %H:%M:%S,%f + file = /usr/lib/ranger/logs/admin/logs/ranger-admin* + buffer_duration = 500 + log_stream_name = adminlog-{instance_id} + initial_position = start_of_file + log_group_name = ${RangerLogGroup} + commands: + startawslogs: + command: !Sub | + sudo yum update -y + sudo yum install -y awslogs + #update for amazon linux 2 + #sudo service awslogs restart + #sudo chkconfig awslogs on + sudo systemctl start awslogsd + sudo systemctl enable awslogsd.service + RangerServer: + commands: + installrangerserver: + command: + !Join ['', ['bash /tmp/install-ranger-admin-server.sh ', + !Ref 'LDAPHostPrivateIP', ' ', !Ref 'LDAPSearchBase', ' ', + !Ref 'LDAPBindUserName', '@',!Ref 'DomainDNSName' , ' ', !Ref 'LDAPBindPassword', + ' ', !Ref 'rangerVersion', ' ', !Join ['', ['s3://', !Ref S3Bucket, + '/', !Ref S3Key]], + ' ', !Ref 'ProjectVersion', + ' ', !Ref 'DBHostName', + ' ', !Ref 'DBRootPassword', + ' '," >\ + \ create-ranger-server-output.log \n"]] + Properties: + SubnetId: !Ref 'Subnet' + IamInstanceProfile: !Ref ManagedInstanceProfile + SecurityGroupIds: + - !If [ AttachAdditionalSourcePrefixToSG, !Ref sgRangerAdminServerWithAdditions, !Ref sgRangerAdminServer ] + ImageId: !FindInMap [AWSRegionArch2AMI, !Ref 'AWS::Region', !FindInMap [AWSInstanceType2Arch, + !Ref 'InstanceType', Arch]] + InstanceType: !Ref 'InstanceType' + KeyName: !Ref 'KeyPairName' + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: + VolumeType: gp2 + VolumeSize: '100' + DeleteOnTermination: 'true' + UserData: !Base64 + Fn::Join: + - '' + - - '#!/bin/bash + + ' + - 'cd /tmp + + ' + - 'aws s3 cp ' + - !Join ['', ['s3://', !Ref S3Bucket, '/', !Ref S3Key, '/', !Ref ProjectVersion]] + - '/scripts/install-ranger-admin-server.sh . + + ' + - 'yum update aws-cfn-bootstrap + + ' + - '# Install the files and packages from the metadata + + ' + - '/opt/aws/bin/cfn-init ' + - ' --stack ' + - !Ref 'AWS::StackName' + - ' --resource myEC2 ' + - ' --configsets ascending ' + - ' --region ' + - !Ref 'AWS::Region' + - ' + + ' + Tags: + - Key: Name + Value: RangerServer +Outputs: + IPAddress: + Description: IP address of the Ranger server + Value: !GetAtt [myEC2, PrivateIp] + PrivateDNS: + Description: IP address of the Ranger server + Value: !GetAtt [myEC2, PrivateDnsName]