From f8a69b602e246fb43dd621bb8e59a2bb7c8e9219 Mon Sep 17 00:00:00 2001 From: Alexander Dobrzhansky Date: Thu, 29 Sep 2022 18:30:58 +0200 Subject: [PATCH 1/2] ec2 managing script --- bin/aws-utils/README.md | 65 ++++++ bin/aws-utils/ecmgm/__init__.py | 2 + bin/aws-utils/ecmgm/__main__.py | 9 + bin/aws-utils/ecmgm/cli.py | 171 ++++++++++++++ bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO | 4 + .../ecmgm/ecmgm.egg-info/SOURCES.txt | 9 + .../ecmgm/ecmgm.egg-info/dependency_links.txt | 1 + .../ecmgm/ecmgm.egg-info/not-zip-safe | 1 + .../ecmgm/ecmgm.egg-info/top_level.txt | 1 + bin/aws-utils/ecmgm/schemas.py | 52 +++++ bin/aws-utils/ecmgm/utils.py | 216 ++++++++++++++++++ bin/aws-utils/pyproject.toml | 3 + bin/aws-utils/setup.cfg | 35 +++ bin/aws-utils/setup.py | 6 + 14 files changed, 575 insertions(+) create mode 100644 bin/aws-utils/README.md create mode 100644 bin/aws-utils/ecmgm/__init__.py create mode 100644 bin/aws-utils/ecmgm/__main__.py create mode 100644 bin/aws-utils/ecmgm/cli.py create mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO create mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt create mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt create mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe create mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt create mode 100644 bin/aws-utils/ecmgm/schemas.py create mode 100644 bin/aws-utils/ecmgm/utils.py create mode 100644 bin/aws-utils/pyproject.toml create mode 100644 bin/aws-utils/setup.cfg create mode 100644 bin/aws-utils/setup.py diff --git a/bin/aws-utils/README.md b/bin/aws-utils/README.md new file mode 100644 index 0000000..692563f --- /dev/null +++ b/bin/aws-utils/README.md @@ -0,0 +1,65 @@ +# EC2 Instance Manager + +# Table of contents + + +- [EC2 Instance Manager](#ec2-instance-manager) +- [Table of contents](#table-of-contents) + - [Python Version](#python-version) + - [Third Party Libraries and Dependencies](#third-party-libraries-and-dependencies) + - [Usage](#usage) + - [Examples](#examples) + + +## Python Version +Python 3.9+ are supported and tested + + +## Third Party Libraries and Dependencies + +The following libraries will be installed when you install the client library: +* [typer](https://github.com/tiangolo/typer) +* [boto3](https://github.com/boto/boto3) + +## Usage + +To start using the library you need to setup a list of ENV variables with your AWS credentials as follows: +```sh +export AWS_DEFAULT_REGION=<...> +export AWS_ACCESS_KEY_ID=<...> +export AWS_SECRET_ACCESS_KEY=<...> +``` +Then you can install a library with pip: + +```sh +pip install . +``` + +If you plan to do a local developent you may also want install it in [editable mode](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#working-in-development-mode). +```sh +pip install --editable . +``` + +Afther the library is installed you can simply start it with --help option for retrieveing the further insructions +```sh +ecmgm --help +``` + +## Examples + +Here is the list of example commands: + +Create a VM, using existing AWS SSH key +```sh +ecmgm create --osnick --ssh-key-name +``` + +Retrieve VM description +```sh +ecmgm describe +``` + +Teardown VM +```sh +ecmgm teardown +``` \ No newline at end of file diff --git a/bin/aws-utils/ecmgm/__init__.py b/bin/aws-utils/ecmgm/__init__.py new file mode 100644 index 0000000..5b9a997 --- /dev/null +++ b/bin/aws-utils/ecmgm/__init__.py @@ -0,0 +1,2 @@ +__app_name__ = "ecmgm" +__version__ = "0.1.0" diff --git a/bin/aws-utils/ecmgm/__main__.py b/bin/aws-utils/ecmgm/__main__.py new file mode 100644 index 0000000..da656a6 --- /dev/null +++ b/bin/aws-utils/ecmgm/__main__.py @@ -0,0 +1,9 @@ +from ecmgm import cli, __app_name__ + + +def main(): + cli.app(prog_name=__app_name__) + + +if __name__ == "__main__": + main() diff --git a/bin/aws-utils/ecmgm/cli.py b/bin/aws-utils/ecmgm/cli.py new file mode 100644 index 0000000..3db3c8a --- /dev/null +++ b/bin/aws-utils/ecmgm/cli.py @@ -0,0 +1,171 @@ +import getpass +from typing import Optional + +import boto3 +import typer +from rich.console import Console, Group +from rich.panel import Panel +from rich.prompt import Confirm +from rich.syntax import Syntax +from rich.table import Table + +from ecmgm import __app_name__, __version__, schemas, utils + +app = typer.Typer() +console = Console() + + +@app.command() +def describe(instance_id: str = typer.Argument(..., help="EC2 Instance Id")): + ec2 = boto3.client("ec2") + table = Table(title="Instance details") + table.add_column("Attribute", style="magenta") + table.add_column("Value", justify="left", style="cyan") + + response = ec2.describe_instances( + InstanceIds=[ + instance_id, + ], + ) + + for k, v in response["Reservations"][0]["Instances"][0].items(): + table.add_row(k, str(v)) + + console.print(table) + + +@app.command() +def teardown(instance_id: str = typer.Argument(..., help="EC2 Instance Id")): + instance = utils.terminate_instance(instance_id) + with console.status("[bold green]Waiting for EC2 Instance being terminated..."): + instance.wait_until_terminated() + + +@app.command() +def create( + name: str = typer.Argument(..., help="Virtual machine name"), + os_image: schemas.OsImage = typer.Argument(..., help="OS image"), + instance_type: schemas.InstanceTypes = typer.Option( + schemas.InstanceTypes.small.value, help="Instance types" + ), + instance_arch: schemas.InstanceArch = typer.Option( + schemas.InstanceArch.x86_64.value, help="Instance Arch" + ), + osnick: str = typer.Option("", help="Optionally filter images by specified osnick"), + ssh_key_name: str = typer.Option( + "", help="You can specify your SSH key name in case its already created in AWS" + ), +): + ec2 = boto3.client("ec2") + image_filter_query = schemas.OS_SEARCH_MAPPING[os_image] + + if osnick: + image_filter_query += f"{osnick}*" + + with console.status( + f"[bold green]Searching for image. Search params: query: {image_filter_query} arch: {instance_arch} osnick: {osnick}..." + ): + images = ec2.describe_images( + Filters=[ + { + "Name": "architecture", + "Values": [ + instance_arch, + ], + }, + {"Name": "root-device-type", "Values": ["ebs"]}, + {"Name": "state", "Values": ["available"]}, + {"Name": "virtualization-type", "Values": ["hvm"]}, + {"Name": "hypervisor", "Values": ["xen"]}, + {"Name": "image-type", "Values": ["machine"]}, + { + "Name": "name", + "Values": [image_filter_query], + }, + ], + Owners=["amazon"], + ) + + sorted_amis = sorted(images["Images"], key=lambda x: x["CreationDate"], reverse=True) + target_image = sorted_amis[0] + panel_group = Group( + Panel( + target_image["ImageId"], + title="ami-id", + ), + Panel(target_image["Description"], title="Description"), + Panel(target_image["ImageLocation"], title="Image"), + ) + console.print("Image found, details:") + console.print(Panel(panel_group)) + + is_continue = Confirm.ask("Do you want to continue?") + if not is_continue: + raise typer.Exit() + + if not ssh_key_name: + console.print(f"No SSH key was specified, creating new") + private_key_name = f"{getpass.getuser()}-ec2-{os_image.value}-key" + private_key_filename = f"{getpass.getuser()}-ec2-{os_image.value}-key-file.pem" + key_pair = utils.create_key_pair(private_key_name, private_key_filename) + console.print(f"Created a key pair [bold cyan]{key_pair.key_name}[/bold cyan]") + + instance = utils.create_instance( + target_image["ImageId"], + name, + instance_type, + ssh_key_name or key_pair.key_name, + ["redis-io-group"], + ) + + with console.status("[bold green]Waiting EC2 Instance to start..."): + instance.wait_until_running() + # updating instance attributes to obtain public ip/dns immediately + instance.reload() + + private_key_filename = private_key_filename if not ssh_key_name else "your-key.pem" + panel_group = Group( + Panel("You can now connect to your ec2 machine :thumbs_up:\n"), + Panel( + Syntax( + f"ssh -i {private_key_filename} {schemas.OS_DEFAULT_USER_MAP[os_image]}@{instance.public_dns_name}", + "shell", + theme="monokai", + ), + title="Command", + ), + Panel( + Syntax( + f"# you also might need to make key to be only readable by you\nchmod 400 {private_key_filename}\n" + f"# You may use that instance id [{instance.id}] in other CLI commands like\n" + "# Get VM details\n" + f"ec2 describe {instance.id}\n" + "# Terminate VM\n" + f"ec2 teardown {instance.id}", + "shell", + theme="monokai", + ), + title="Tip", + ), + ) + console.print(Panel(panel_group)) + + +def _version_callback(value: bool) -> None: + if value: + typer.echo(f"{__app_name__} v{__version__}") + raise typer.Exit() + + +@app.callback() +def main( + version: Optional[bool] = typer.Option( + None, + "--version", + "-v", + help="Show the application's version and exit.", + callback=_version_callback, + is_eager=True, + ) +) -> None: + return diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO b/bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO new file mode 100644 index 0000000..c13adc6 --- /dev/null +++ b/bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO @@ -0,0 +1,4 @@ +Metadata-Version: 2.1 +Name: ecmgm +Version: 0.1.0 +Classifier: Programming Language :: Python :: 3 diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt b/bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt new file mode 100644 index 0000000..433fa76 --- /dev/null +++ b/bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt @@ -0,0 +1,9 @@ +README.md +pyproject.toml +setup.cfg +setup.py +ecmgm/ecmgm.egg-info/PKG-INFO +ecmgm/ecmgm.egg-info/SOURCES.txt +ecmgm/ecmgm.egg-info/dependency_links.txt +ecmgm/ecmgm.egg-info/not-zip-safe +ecmgm/ecmgm.egg-info/top_level.txt \ No newline at end of file diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt b/bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe b/bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt b/bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt @@ -0,0 +1 @@ + diff --git a/bin/aws-utils/ecmgm/schemas.py b/bin/aws-utils/ecmgm/schemas.py new file mode 100644 index 0000000..0894fb5 --- /dev/null +++ b/bin/aws-utils/ecmgm/schemas.py @@ -0,0 +1,52 @@ +from enum import Enum + + +class InstanceTypes(str, Enum): + """ + Instance vCPU* Mem (GiB) + t2.nano 1 0.5 + t2.micro 1 1 + t2.small 1 2 + t2.medium 2 4 + t2.large 2 8 + t2.xlarge 4 16 + t2.2xlarge 8 32 + """ + + nano = "t2.nano" + micro = "t2.micro" + small = "t2.small" + medium = "t2.medium" + large = "t2.large" + xlarge = "t2.xlarge" + doublelarge = "t2.2xlarge" + + +class InstanceArch(str, Enum): + i386 = "i386" + x86_64 = "x86_64" + x86_64_mac = "x86_64_mac" + arm64 = "arm64" + + +class OsImage(str, Enum): + windows = "windows" + ubuntu = "ubuntu" + debian = "debian" + suse = "suse" + amazon_linux = "amazon_linux" + redhat = "redhat" + macos = "macos" + + +OS_SEARCH_MAPPING = { + OsImage.windows: "*Windows*", + OsImage.ubuntu: "*ubuntu*/images/*", + OsImage.debian: "*debian", + OsImage.suse: "*suse*", + OsImage.amazon_linux: "amzn2-ami-hvm-*", + OsImage.redhat: "*RHEL*", + OsImage.macos: "*macos*", +} + +OS_DEFAULT_USER_MAP = {OsImage.ubuntu: "ubuntu", OsImage.amazon_linux: "ec2"} diff --git a/bin/aws-utils/ecmgm/utils.py b/bin/aws-utils/ecmgm/utils.py new file mode 100644 index 0000000..bcd9387 --- /dev/null +++ b/bin/aws-utils/ecmgm/utils.py @@ -0,0 +1,216 @@ +import base64 +import os +import struct + +import boto3 +import paramiko +from botocore.exceptions import ClientError +from paramiko.util import deflate_long +from rich.console import Console + +console = Console() +ec2 = boto3.resource("ec2") + + +def start_instance(instance_id): + try: + response = ec2.Instance(instance_id).start() + console.print(f"Started instance {instance_id}") + except ClientError: + console.print(f"Couldn't start instance {instance_id}") + raise + else: + return response + + +def stop_instance(instance_id): + try: + response = ec2.Instance(instance_id).stop() + console.print(f"Stopped instance {instance_id}") + except ClientError: + console.print(f"Couldn't stop instance {instance_id}") + raise + else: + return response + + +def get_console_output(instance_id): + try: + output = ec2.Instance(instance_id).console_output()["Output"] + console.print(f"Got console output for instance {instance_id}") + except ClientError: + console.print((f"Couldn't get console output for instance {instance_id}")) + raise + else: + return output + + +def import_key_pair(key_name: str, private_key_file_path: str) -> ec2.KeyPair: + key = paramiko.RSAKey.from_private_key_file(private_key_file_path) + + output = b"" + parts = [ + b"ssh-rsa", + deflate_long(key.public_numbers.e), + deflate_long(key.public_numbers.n), + ] + + for part in parts: + output += struct.pack(">I", len(part)) + part + public_key = b"ssh-rsa " + base64.b64encode(output) + b"\n" + + key_pair = ec2.import_key_pair(KeyName=key_name, PublicKeyMaterial=public_key) + return key_pair + + +def create_key_pair(key_name: str, private_key_file_name: str = None) -> ec2.KeyPair: + try: + key_pair = ec2.create_key_pair(KeyName=key_name) + console.print(f"Created key [bold cyan]{key_pair.name}[/bold cyan].") + if private_key_file_name is not None: + with open(private_key_file_name, "w") as pk_file: + pk_file.write(key_pair.key_material) + console.print( + f"Wrote private key to [bold cyan]{private_key_file_name}[/bold cyan]." + ) + except ClientError: + console.print(f"Couldn't create key {key_name}.") + raise + else: + return key_pair + + +def setup_security_group(group_name, group_description, ssh_ingress_ip=None): + try: + default_vpc = list( + ec2.vpcs.filter(Filters=[{"Name": "isDefault", "Values": ["true"]}]) + )[0] + console.print(f"Got default VPC {default_vpc.id}") + except ClientError: + console.print("Couldn't get VPCs.") + raise + except IndexError: + console.print("No default VPC in the list.") + raise + + try: + security_group = default_vpc.create_security_group( + GroupName=group_name, Description=group_description + ) + console.print(f"Created security group {group_name} in VPC {default_vpc.id}.") + except ClientError: + console.print(f"Couldn't create security group {group_name}.") + raise + + try: + ip_permissions = [ + { + # HTTP ingress open to anyone + "IpProtocol": "tcp", + "FromPort": 80, + "ToPort": 80, + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + }, + { + # HTTPS ingress open to anyone + "IpProtocol": "tcp", + "FromPort": 443, + "ToPort": 443, + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + }, + ] + if ssh_ingress_ip is not None: + ip_permissions.append( + { + # SSH ingress open to only the specified IP address + "IpProtocol": "tcp", + "FromPort": 22, + "ToPort": 22, + "IpRanges": [{"CidrIp": f"{ssh_ingress_ip}/32"}], + } + ) + security_group.authorize_ingress(IpPermissions=ip_permissions) + console.print( + f"Set inbound rules for {security_group.id} to allow all inbound HTTP and HTTPS " + f"but only {ssh_ingress_ip} for SSH." + ) + except ClientError: + console.print(f"Couldnt authorize inbound rules for {group_name}.") + raise + else: + return security_group + + +def create_instance( + image_id: str, + name: str, + instance_type: str, + key_name: str, + security_group_names: list = None, +): + try: + instance_params = { + "ImageId": image_id, + "InstanceType": instance_type, + "KeyName": key_name, + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + {"Key": "Name", "Value": name}, + ], + }, + ], + } + if security_group_names is not None: + instance_params["SecurityGroups"] = security_group_names + instance = ec2.create_instances(**instance_params, MinCount=1, MaxCount=1)[0] + console.print(f"Created instance [bold cyan]{instance.id}[/bold cyan].") + except ClientError: + console.print( + f"Couldn't create instance with image {image_id}, instance type {instance_type}, and key {key_name}." + ) + raise + else: + return instance + + +def delete_key_pair(key_name: str, key_file_name: str) -> None: + """ + Deletes a key pair and the specified private key file. + """ + try: + ec2.KeyPair(key_name).delete() + os.remove(key_file_name) + console.print(f"Deleted key {key_name} and private key file {key_file_name}.") + except ClientError: + console.print(f"Couldn't delete key {key_name}.") + raise + + +def delete_security_group(group_id): + """ + Deletes a security group. + """ + try: + ec2.SecurityGroup(group_id).delete() + console.print(f"Deleted security group {group_id}.") + except ClientError: + console.print( + f"Couldn't delete security group {group_id}.", + ) + raise + + +def terminate_instance(instance_id): + """ + Terminates an instance. The request returns immediately. + """ + try: + instance = ec2.Instance(instance_id) + instance.terminate() + console.print(f"Terminating instance {instance_id}.") + return instance + except ClientError: + console.print(f"Couldn't terminate instance {instance_id}.") + raise diff --git a/bin/aws-utils/pyproject.toml b/bin/aws-utils/pyproject.toml new file mode 100644 index 0000000..bf91296 --- /dev/null +++ b/bin/aws-utils/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "wheel"] diff --git a/bin/aws-utils/setup.cfg b/bin/aws-utils/setup.cfg new file mode 100644 index 0000000..81d6733 --- /dev/null +++ b/bin/aws-utils/setup.cfg @@ -0,0 +1,35 @@ +[metadata] +name = ecmgm +version = attr: ecmgm.__version__ +classifiers = + Programming Language :: Python :: 3 + +[options] +zip_safe = False +include_package_data = True +packages=find: +install_requires = + bcrypt ==4.0.0 + boto3 ==1.24.70 + botocore ==1.27.70 + cffi ==1.15.1 + click ==8.1.3 + colorama ==0.4.5 + commonmark ==0.9.1 + cryptography ==38.0.1 + jmespath ==1.0.1 + paramiko ==2.11.0 + pycparser ==2.21 + Pygments ==2.13.0 + PyNaCl ==1.5.0 + python-dateutil ==2.8.2 + rich ==12.5.1 + s3transfer ==0.6.0 + shellingham ==1.5.0 + six ==1.16.0 + typer ==0.6.1 + urllib3 ==1.26.12 + +[options.entry_points] +console_scripts = + ecmgm = ecmgm.__main__:main diff --git a/bin/aws-utils/setup.py b/bin/aws-utils/setup.py new file mode 100644 index 0000000..3762150 --- /dev/null +++ b/bin/aws-utils/setup.py @@ -0,0 +1,6 @@ +import setuptools + +if __name__ == '__main__': + setuptools.setup( + # see 'setup.cfg' + ) From 530e35bbefe51830208844fddf766e6c5d3fe51b Mon Sep 17 00:00:00 2001 From: Alexander Dobrzhansky Date: Thu, 13 Oct 2022 17:44:18 +0200 Subject: [PATCH 2/2] add macos support --- .gitignore | 17 ++++ bin/aws-utils/ecmgm/cli.py | 74 ++++++++++++++-- bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO | 4 - .../ecmgm/ecmgm.egg-info/SOURCES.txt | 9 -- .../ecmgm/ecmgm.egg-info/dependency_links.txt | 1 - .../ecmgm/ecmgm.egg-info/not-zip-safe | 1 - .../ecmgm/ecmgm.egg-info/top_level.txt | 1 - bin/aws-utils/ecmgm/schemas.py | 27 ++++-- bin/aws-utils/ecmgm/utils.py | 84 +++++++++++++------ 9 files changed, 161 insertions(+), 57 deletions(-) delete mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO delete mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt delete mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt delete mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe delete mode 100644 bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt diff --git a/.gitignore b/.gitignore index 5cc2872..c224826 100755 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,18 @@ *.pyc *.pyo +# Distribution / packaging +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +parts/ +sdist/ +*.egg-info/ +.installed.cfg +*.egg + # CMake CMakeFiles/ *.cmake @@ -39,4 +51,9 @@ CMakeCache.txt .cproject .project +# VScode +.vscode/ + venv/ +.venv/ + diff --git a/bin/aws-utils/ecmgm/cli.py b/bin/aws-utils/ecmgm/cli.py index 3db3c8a..bd72807 100644 --- a/bin/aws-utils/ecmgm/cli.py +++ b/bin/aws-utils/ecmgm/cli.py @@ -15,6 +15,38 @@ console = Console() +@app.command() +def listvm(): + ec2 = boto3.client("ec2") + response = ec2.describe_instances() + table = Table(title="Instance list") + + for column in [ + "Tags", + "ImageId", + "InstanceId", + "InstanceType", + "LaunchTime", + "Monitoring", + "PublicDnsName", + ]: + table.add_column(column, style="magenta") + + for r in response["Reservations"]: + for i in r["Instances"]: + table.add_row( + i["Tags"][0]["Value"], + i["ImageId"], + i["InstanceId"], + i["InstanceType"], + str(i["LaunchTime"]), + str(i["Monitoring"]), + i["PublicDnsName"], + ) + + console.print(table) + + @app.command() def describe(instance_id: str = typer.Argument(..., help="EC2 Instance Id")): ec2 = boto3.client("ec2") @@ -55,7 +87,25 @@ def create( ssh_key_name: str = typer.Option( "", help="You can specify your SSH key name in case its already created in AWS" ), + host_id: str = typer.Option( + "", + help="Dedicated host id (in case you are spinning macos instance) if not provided will be created by default", + ), ): + if ( + instance_type == schemas.InstanceTypes.macmetal.value + and instance_arch != schemas.InstanceArch.x86_64_mac.value + ): + raise ValueError( + f"Unsoported instance_arch, mac instance types supports only {schemas.InstanceArch.x86_64_mac.value}" + ) + if ( + instance_arch == schemas.InstanceArch.x86_64_mac.value + and instance_type != schemas.InstanceTypes.macmetal.value + ): + raise ValueError( + f"Unsoported instance type, mac instance arch supports only {schemas.InstanceTypes.macmetal.value}" + ) ec2 = boto3.client("ec2") image_filter_query = schemas.OS_SEARCH_MAPPING[os_image] @@ -86,7 +136,9 @@ def create( Owners=["amazon"], ) - sorted_amis = sorted(images["Images"], key=lambda x: x["CreationDate"], reverse=True) + sorted_amis = sorted( + images["Images"], key=lambda x: x["CreationDate"], reverse=True + ) target_image = sorted_amis[0] panel_group = Group( Panel( @@ -110,15 +162,25 @@ def create( key_pair = utils.create_key_pair(private_key_name, private_key_filename) console.print(f"Created a key pair [bold cyan]{key_pair.key_name}[/bold cyan]") + if instance_type == schemas.InstanceTypes.macmetal.value and not host_id: + console.print( + f"Creating dedicated host for instance type: [bold cyan]{instance_type}[/bold cyan]" + ) + host_id = utils.allocate_hosts(instance_type) + console.print( + f"Dedicated host with id [bold cyan]{host_id}[/bold cyan] created" + ) + instance = utils.create_instance( target_image["ImageId"], name, instance_type, ssh_key_name or key_pair.key_name, - ["redis-io-group"], + ["redis-io-group"], # todo default security group is temporary hardcoded + host_id, ) - with console.status("[bold green]Waiting EC2 Instance to start..."): + with console.status("[bold green]Waiting for EC2 Instance to start..."): instance.wait_until_running() # updating instance attributes to obtain public ip/dns immediately instance.reload() @@ -137,11 +199,11 @@ def create( Panel( Syntax( f"# you also might need to make key to be only readable by you\nchmod 400 {private_key_filename}\n" - f"# You may use that instance id [{instance.id}] in other CLI commands like\n" + f"# You may use that instance id -> [{instance.id} <- in other CLI commands like\n" "# Get VM details\n" - f"ec2 describe {instance.id}\n" + f"ecmgm describe {instance.id}\n" "# Terminate VM\n" - f"ec2 teardown {instance.id}", + f"ecmgm teardown {instance.id}", "shell", theme="monokai", ), diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO b/bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO deleted file mode 100644 index c13adc6..0000000 --- a/bin/aws-utils/ecmgm/ecmgm.egg-info/PKG-INFO +++ /dev/null @@ -1,4 +0,0 @@ -Metadata-Version: 2.1 -Name: ecmgm -Version: 0.1.0 -Classifier: Programming Language :: Python :: 3 diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt b/bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt deleted file mode 100644 index 433fa76..0000000 --- a/bin/aws-utils/ecmgm/ecmgm.egg-info/SOURCES.txt +++ /dev/null @@ -1,9 +0,0 @@ -README.md -pyproject.toml -setup.cfg -setup.py -ecmgm/ecmgm.egg-info/PKG-INFO -ecmgm/ecmgm.egg-info/SOURCES.txt -ecmgm/ecmgm.egg-info/dependency_links.txt -ecmgm/ecmgm.egg-info/not-zip-safe -ecmgm/ecmgm.egg-info/top_level.txt \ No newline at end of file diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt b/bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/bin/aws-utils/ecmgm/ecmgm.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe b/bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe deleted file mode 100644 index 8b13789..0000000 --- a/bin/aws-utils/ecmgm/ecmgm.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt b/bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt deleted file mode 100644 index 8b13789..0000000 --- a/bin/aws-utils/ecmgm/ecmgm.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/bin/aws-utils/ecmgm/schemas.py b/bin/aws-utils/ecmgm/schemas.py index 0894fb5..956b83c 100644 --- a/bin/aws-utils/ecmgm/schemas.py +++ b/bin/aws-utils/ecmgm/schemas.py @@ -4,13 +4,15 @@ class InstanceTypes(str, Enum): """ Instance vCPU* Mem (GiB) - t2.nano 1 0.5 - t2.micro 1 1 - t2.small 1 2 - t2.medium 2 4 - t2.large 2 8 - t2.xlarge 4 16 - t2.2xlarge 8 32 + t2.nano 1 0.5 + t2.micro 1 1 + t2.small 1 2 + t2.medium 2 4 + t2.large 2 8 + t2.xlarge 4 16 + t2.2xlarge 8 32 + mac1.metal 12 32 + mac2.metal 12 16 """ nano = "t2.nano" @@ -20,6 +22,7 @@ class InstanceTypes(str, Enum): large = "t2.large" xlarge = "t2.xlarge" doublelarge = "t2.2xlarge" + macmetal = "mac1.metal" class InstanceArch(str, Enum): @@ -48,5 +51,11 @@ class OsImage(str, Enum): OsImage.redhat: "*RHEL*", OsImage.macos: "*macos*", } - -OS_DEFAULT_USER_MAP = {OsImage.ubuntu: "ubuntu", OsImage.amazon_linux: "ec2"} +OS_DEFAULT_USER_MAP = { + OsImage.ubuntu: "ubuntu", + OsImage.debian: "admin", + OsImage.redhat: "ec2-user", + OsImage.suse: "ec2-user", + OsImage.amazon_linux: "ec2-user", + OsImage.macos: "ec2-user", +} diff --git a/bin/aws-utils/ecmgm/utils.py b/bin/aws-utils/ecmgm/utils.py index bcd9387..21002b7 100644 --- a/bin/aws-utils/ecmgm/utils.py +++ b/bin/aws-utils/ecmgm/utils.py @@ -10,31 +10,32 @@ console = Console() ec2 = boto3.resource("ec2") +client = boto3.client("ec2") -def start_instance(instance_id): +def start_instance(instance_id: str) -> ec2.Instance: try: - response = ec2.Instance(instance_id).start() + instance = ec2.Instance(instance_id).start() console.print(f"Started instance {instance_id}") except ClientError: console.print(f"Couldn't start instance {instance_id}") raise else: - return response + return instance -def stop_instance(instance_id): +def stop_instance(instance_id: str) -> ec2.Instance: try: - response = ec2.Instance(instance_id).stop() + instance = ec2.Instance(instance_id).stop() console.print(f"Stopped instance {instance_id}") except ClientError: console.print(f"Couldn't stop instance {instance_id}") raise else: - return response + return instance -def get_console_output(instance_id): +def get_console_output(instance_id: str): try: output = ec2.Instance(instance_id).console_output()["Output"] console.print(f"Got console output for instance {instance_id}") @@ -46,6 +47,9 @@ def get_console_output(instance_id): def import_key_pair(key_name: str, private_key_file_path: str) -> ec2.KeyPair: + """ + Import existing key pair in AWS to allow SSH access + """ key = paramiko.RSAKey.from_private_key_file(private_key_file_path) output = b"" @@ -64,6 +68,9 @@ def import_key_pair(key_name: str, private_key_file_path: str) -> ec2.KeyPair: def create_key_pair(key_name: str, private_key_file_name: str = None) -> ec2.KeyPair: + """ + Create key pair in AWS to allow SSH access + """ try: key_pair = ec2.create_key_pair(KeyName=key_name) console.print(f"Created key [bold cyan]{key_pair.name}[/bold cyan].") @@ -80,7 +87,10 @@ def create_key_pair(key_name: str, private_key_file_name: str = None) -> ec2.Key return key_pair -def setup_security_group(group_name, group_description, ssh_ingress_ip=None): +def setup_security_group(group_name: str, group_description: str) -> ec2.SecurityGroup: + """ + Create security group + """ try: default_vpc = list( ec2.vpcs.filter(Filters=[{"Name": "isDefault", "Values": ["true"]}]) @@ -118,22 +128,17 @@ def setup_security_group(group_name, group_description, ssh_ingress_ip=None): "ToPort": 443, "IpRanges": [{"CidrIp": "0.0.0.0/0"}], }, + { + # SSH ingress open to anyone + "IpProtocol": "tcp", + "FromPort": 22, + "ToPort": 22, + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + }, ] - if ssh_ingress_ip is not None: - ip_permissions.append( - { - # SSH ingress open to only the specified IP address - "IpProtocol": "tcp", - "FromPort": 22, - "ToPort": 22, - "IpRanges": [{"CidrIp": f"{ssh_ingress_ip}/32"}], - } - ) + security_group.authorize_ingress(IpPermissions=ip_permissions) - console.print( - f"Set inbound rules for {security_group.id} to allow all inbound HTTP and HTTPS " - f"but only {ssh_ingress_ip} for SSH." - ) + except ClientError: console.print(f"Couldnt authorize inbound rules for {group_name}.") raise @@ -147,7 +152,11 @@ def create_instance( instance_type: str, key_name: str, security_group_names: list = None, -): + host_id: str = None, +) -> ec2.Instance: + """ + Create instance + """ try: instance_params = { "ImageId": image_id, @@ -162,6 +171,11 @@ def create_instance( }, ], } + if host_id: + instance_params["Placement"] = { + "HostId": host_id, + # "Tenancy": "default" | "dedicated" | "host", + } if security_group_names is not None: instance_params["SecurityGroups"] = security_group_names instance = ec2.create_instances(**instance_params, MinCount=1, MaxCount=1)[0] @@ -188,7 +202,7 @@ def delete_key_pair(key_name: str, key_file_name: str) -> None: raise -def delete_security_group(group_id): +def delete_security_group(group_id: str) -> None: """ Deletes a security group. """ @@ -202,15 +216,33 @@ def delete_security_group(group_id): raise -def terminate_instance(instance_id): +def terminate_instance(instance_id: str) -> ec2.Instance: """ Terminates an instance. The request returns immediately. """ try: instance = ec2.Instance(instance_id) instance.terminate() - console.print(f"Terminating instance {instance_id}.") + console.print(f"Terminating instance [bold cyan]{instance_id}[/bold cyan].") return instance except ClientError: console.print(f"Couldn't terminate instance {instance_id}.") raise + + +def allocate_hosts(instance_type: str) -> list: + """ + Create dedicated host for aws default region + """ + try: + response = client.allocate_hosts( + AutoPlacement="on", + # available zones ussualy are aws region + [a,b,c] postfix + AvailabilityZone=os.environ.get("AWS_DEFAULT_REGION") + "a", + InstanceType=instance_type, + Quantity=1, + ) + except ClientError: + console.print(f"Couldn't allocate dedicated host.") + raise + return response["HostIds"][0]