From 1662f1933eb5f379ee24a1e2d96b8cc3eb30ff4d Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 14 Feb 2023 16:36:15 +0100 Subject: [PATCH 1/8] Add initial support for packer --- .gitignore | 2 + ansible/apply | 38 ++++++------ packer/README.md | 15 +++++ packer/docs-rs-builder/builder.pkr.hcl | 45 ++++++++++++++ packer/docs-rs-builder/variables.pkr.hcl | 4 ++ packer/packer | 75 ++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 packer/README.md create mode 100644 packer/docs-rs-builder/builder.pkr.hcl create mode 100644 packer/docs-rs-builder/variables.pkr.hcl create mode 100755 packer/packer diff --git a/.gitignore b/.gitignore index ed5c66cf6..90a232fc8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ /target /ansible/.venv /ansible/envs/dev +/packer/.workspace /terraform/shared/modules/lambda/packages /terraform/rds-databases/.forward-ports-cache-*.json /terragrunt/modules/aws-lambda/packages .terraform +packer-provisioner-* node_modules __pycache__ *.py[co] diff --git a/ansible/apply b/ansible/apply index f2147dd65..0d2533ce2 100755 --- a/ansible/apply +++ b/ansible/apply @@ -13,9 +13,9 @@ VENV_PATH = BASE_PATH / ".venv" # Ansible changes a lot between releases and deprecates a lot of stuff each of # them. Using a pinned ansible identical between all team members should # reduce the churn. -def install_ansible(): +def install_ansible(venv_path = VENV_PATH): requirements = BASE_PATH / "requirements.txt" - venv_requirements = VENV_PATH / "installed-requirements.txt" + venv_requirements = venv_path / "installed-requirements.txt" # Avoid installing ansible in the virtualenv multiple times if venv_requirements.exists() and \ @@ -23,29 +23,31 @@ def install_ansible(): return print("creating a new virtual environment and install ansible in it...") - shutil.rmtree(VENV_PATH, ignore_errors=True) - subprocess.run([sys.executable, "-m", "venv", str(VENV_PATH)], check=True) + shutil.rmtree(venv_path, ignore_errors=True) + subprocess.run([sys.executable, "-m", "venv", str(venv_path)], check=True) subprocess.run([ - str(VENV_PATH / "bin" / "pip"), "install", "-r", str(requirements), + str(venv_path / "bin" / "pip"), "install", "-r", str(requirements), ], check=True) shutil.copy(str(requirements), str(venv_requirements)) +def create_workspace(dir, env, playbook): + env_dir = BASE_PATH / "envs" / env + # Create a temporary directory merging together the chosen + # environment, the chosen playbook and the shared files. + (dir / "play").mkdir() + (dir / "play" / "roles").symlink_to(BASE_PATH / "roles") + (dir / "play" / "group_vars").symlink_to(BASE_PATH / "group_vars") + (dir / "play" / "playbook.yml").symlink_to( + BASE_PATH / "playbooks" / (playbook + ".yml") + ) + (dir / "env").symlink_to(env_dir) + (dir / "ansible.cfg").symlink_to(BASE_PATH / "ansible.cfg") + def run_playbook(args): - env_dir = BASE_PATH / "envs" / args.env tempdir = pathlib.Path(tempfile.mkdtemp()) + create_workspace(tempdir, args.env, args.playbook) try: - # Create a temporary directory merging together the chosen - # environment, the chosen playbook and the shared files. - (tempdir / "play").mkdir() - (tempdir / "play" / "roles").symlink_to(BASE_PATH / "roles") - (tempdir / "play" / "group_vars").symlink_to(BASE_PATH / "group_vars") - (tempdir / "play" / "playbook.yml").symlink_to( - BASE_PATH / "playbooks" / (args.playbook + ".yml") - ) - (tempdir / "env").symlink_to(env_dir) - (tempdir / "ansible.cfg").symlink_to(BASE_PATH / "ansible.cfg") - - # Finally invoke the ansible binary installed in the virtualenv + # Invoke the ansible binary installed in the virtualenv ansible_args = [ str(VENV_PATH / "bin" / "ansible-playbook"), "-i", str(tempdir / "env" / "hosts"), diff --git a/packer/README.md b/packer/README.md new file mode 100644 index 000000000..3b995bf91 --- /dev/null +++ b/packer/README.md @@ -0,0 +1,15 @@ +# Packer + +This directory contains configuration for running packer to build AMIs. + +## The `packer` wrapper + +This directory contains a python script named `packer` which wraps the system `packer`. The script creates an ansible virtual environment and moves the correct ansible configuration in place. + +### Running `packer` + +To run the wrapper pass the environment, playbook, and any vars ansible is expecting: + +```bash +$ ./packer staging docs-rs-builder --var "revision=SOME_REVISION5 +``` diff --git a/packer/docs-rs-builder/builder.pkr.hcl b/packer/docs-rs-builder/builder.pkr.hcl new file mode 100644 index 000000000..c78146d21 --- /dev/null +++ b/packer/docs-rs-builder/builder.pkr.hcl @@ -0,0 +1,45 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.1.1" + source = "github.com/hashicorp/amazon" + } + } +} + +source "amazon-ebs" "ubuntu" { + ami_name = "docs-rs-builder-${var.revision}" + instance_type = "t2.large" + region = "us-east-1" + source_ami_filter { + filters = { + name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + } + ssh_username = "ubuntu" + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 64 + delete_on_termination = true + } +} + +build { + sources = [ + "source.amazon-ebs.ubuntu" + ] + + provisioner "ansible" { + command = ".venv/bin/ansible-playbook" + # The default is "default" + host_alias = "docs-rs-builder" + inventory_directory = "./env" + playbook_file = "./play/playbook.yml" + # The default is the user running packer + user = "ubuntu" + } +} diff --git a/packer/docs-rs-builder/variables.pkr.hcl b/packer/docs-rs-builder/variables.pkr.hcl new file mode 100644 index 000000000..49928dd2c --- /dev/null +++ b/packer/docs-rs-builder/variables.pkr.hcl @@ -0,0 +1,4 @@ +variable "revision" { + type = string + description = "the revision of the builder to build" +} \ No newline at end of file diff --git a/packer/packer b/packer/packer new file mode 100755 index 000000000..8480d19a6 --- /dev/null +++ b/packer/packer @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import os +import sys +import subprocess +import pathlib +import shutil +import argparse + +# Python makes us jump through hoops to import module from +# a file that does not end in .py +def load_module(): + from importlib.util import spec_from_loader, module_from_spec + from importlib.machinery import SourceFileLoader + apply = os.path.join(os.path.dirname(__file__), '../ansible/apply') + spec = spec_from_loader("apply", SourceFileLoader("apply", apply)) + apply = module_from_spec(spec) + spec.loader.exec_module(apply) + return apply + +# Utility for removing everything in a directory except one thing +def remove_all_except_one(directory, exception): + for item in os.listdir(directory): + if item != exception: + item_path = os.path.join(directory, item) + if os.path.islink(item_path) or os.path.isfile(item_path): + # Delete the file or symlink + os.remove(item_path) + else: + # Delete the directory + shutil.rmtree(item_path) + +# Create the workspace directory leaving virtual environment +# if it's already there +def create_workspace_dir(): + dir = pathlib.Path(__file__).resolve().parent + workspace = os.path.join(dir, '.workspace') + if os.path.exists(workspace): + # Clean up workspace except for virtual environment + remove_all_except_one(workspace, ".venv") + else: + # Make workspace + os.mkdir(workspace) + return pathlib.Path(workspace) + +# Create the workspace environment +def create_workspace(env, playbook): + workspace = create_workspace_dir() + apply = load_module() + apply.install_ansible(workspace / ".venv") + apply.create_workspace(workspace, env, playbook) + + # Link the template into the workspace + template_path = pathlib.Path(playbook).resolve() + if not os.path.exists(template_path): + raise Exception(f"Last argument to packer call was the file '{template_path}' which does not exist") + (workspace / "template").symlink_to(template_path) + return workspace + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("env") + parser.add_argument("playbook") + parser.add_argument('--var', action='append', help='Ansible variables as $key=$value') + args = parser.parse_args() + + workspace = create_workspace(args.env, args.playbook) + # Call packer with the right arguments from the workspace + vars = [] + for var in args.var: + vars += ["-var", var] + cmd = ["packer", "build", *vars, "template"] + res = subprocess.run(cmd, cwd=str(workspace)) + if res.returncode != 0: + exit(1) From 3f245eb8f8460b691d43140a14842ab6b7c572fd Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Tue, 14 Feb 2023 17:32:16 +0100 Subject: [PATCH 2/8] Don't reboot --- ansible/roles/common/tasks/networking.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/common/tasks/networking.yml b/ansible/roles/common/tasks/networking.yml index ca8901583..d701042ed 100644 --- a/ansible/roles/common/tasks/networking.yml +++ b/ansible/roles/common/tasks/networking.yml @@ -3,7 +3,7 @@ - name: set the system hostname hostname: name: "{{ inventory_hostname }}" - notify: reboot + # notify: reboot - name: upload /etc/hosts template: From 3ee6f80cdacb25e5411e7b1fd43eb3875cef61df Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 15 Feb 2023 10:45:12 +0100 Subject: [PATCH 3/8] Move utils to separate file --- ansible/ansible.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ ansible/apply | 45 ++++----------------------------------------- packer/packer | 20 +++++--------------- 3 files changed, 53 insertions(+), 56 deletions(-) create mode 100755 ansible/ansible.py diff --git a/ansible/ansible.py b/ansible/ansible.py new file mode 100755 index 000000000..41a75e1f6 --- /dev/null +++ b/ansible/ansible.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Utilities for creating the Ansible environment we use. + +import subprocess +import pathlib +import sys +import shutil + +BASE_PATH = pathlib.Path(__file__).resolve().parent +VENV_PATH = BASE_PATH / ".venv" + +# Ansible changes a lot between releases and deprecates a lot of stuff each of +# them. Using a pinned ansible identical between all team members should +# reduce the churn. +def install_ansible(venv_path = VENV_PATH): + requirements = BASE_PATH / "requirements.txt" + venv_requirements = venv_path / "installed-requirements.txt" + + # Avoid installing ansible in the virtualenv multiple times + if venv_requirements.exists() and \ + venv_requirements.read_bytes() == requirements.read_bytes(): + return + + print("creating a new virtual environment and install ansible in it...") + shutil.rmtree(venv_path, ignore_errors=True) + subprocess.run([sys.executable, "-m", "venv", str(venv_path)], check=True) + subprocess.run([ + str(venv_path / "bin" / "pip"), "install", "-r", str(requirements), + ], check=True) + shutil.copy(str(requirements), str(venv_requirements)) + +def create_workspace(dir, env, playbook): + env_dir = BASE_PATH / "envs" / env + # Create a temporary directory merging together the chosen + # environment, the chosen playbook and the shared files. + (dir / "play").mkdir() + (dir / "play" / "roles").symlink_to(BASE_PATH / "roles") + (dir / "play" / "group_vars").symlink_to(BASE_PATH / "group_vars") + (dir / "play" / "playbook.yml").symlink_to( + BASE_PATH / "playbooks" / (playbook + ".yml") + ) + (dir / "env").symlink_to(env_dir) + (dir / "ansible.cfg").symlink_to(BASE_PATH / "ansible.cfg") diff --git a/ansible/apply b/ansible/apply index 0d2533ce2..7ee256935 100755 --- a/ansible/apply +++ b/ansible/apply @@ -1,55 +1,18 @@ #!/usr/bin/env python3 import subprocess import pathlib -import sys import shutil import tempfile import argparse -import os - -BASE_PATH = pathlib.Path(__file__).resolve().parent -VENV_PATH = BASE_PATH / ".venv" - -# Ansible changes a lot between releases and deprecates a lot of stuff each of -# them. Using a pinned ansible identical between all team members should -# reduce the churn. -def install_ansible(venv_path = VENV_PATH): - requirements = BASE_PATH / "requirements.txt" - venv_requirements = venv_path / "installed-requirements.txt" - - # Avoid installing ansible in the virtualenv multiple times - if venv_requirements.exists() and \ - venv_requirements.read_bytes() == requirements.read_bytes(): - return - - print("creating a new virtual environment and install ansible in it...") - shutil.rmtree(venv_path, ignore_errors=True) - subprocess.run([sys.executable, "-m", "venv", str(venv_path)], check=True) - subprocess.run([ - str(venv_path / "bin" / "pip"), "install", "-r", str(requirements), - ], check=True) - shutil.copy(str(requirements), str(venv_requirements)) - -def create_workspace(dir, env, playbook): - env_dir = BASE_PATH / "envs" / env - # Create a temporary directory merging together the chosen - # environment, the chosen playbook and the shared files. - (dir / "play").mkdir() - (dir / "play" / "roles").symlink_to(BASE_PATH / "roles") - (dir / "play" / "group_vars").symlink_to(BASE_PATH / "group_vars") - (dir / "play" / "playbook.yml").symlink_to( - BASE_PATH / "playbooks" / (playbook + ".yml") - ) - (dir / "env").symlink_to(env_dir) - (dir / "ansible.cfg").symlink_to(BASE_PATH / "ansible.cfg") +import ansible def run_playbook(args): tempdir = pathlib.Path(tempfile.mkdtemp()) - create_workspace(tempdir, args.env, args.playbook) + ansible.create_workspace(tempdir, args.env, args.playbook) try: # Invoke the ansible binary installed in the virtualenv ansible_args = [ - str(VENV_PATH / "bin" / "ansible-playbook"), + str(ansible.VENV_PATH / "bin" / "ansible-playbook"), "-i", str(tempdir / "env" / "hosts"), str(tempdir / "play" / "playbook.yml"), ] @@ -76,5 +39,5 @@ if __name__ == "__main__": ) args = parser.parse_args() - install_ansible() + ansible.install_ansible() run_playbook(args) diff --git a/packer/packer b/packer/packer index 8480d19a6..b0864fc9d 100755 --- a/packer/packer +++ b/packer/packer @@ -1,22 +1,13 @@ #!/usr/bin/env python3 import os -import sys import subprocess import pathlib import shutil import argparse - -# Python makes us jump through hoops to import module from -# a file that does not end in .py -def load_module(): - from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import SourceFileLoader - apply = os.path.join(os.path.dirname(__file__), '../ansible/apply') - spec = spec_from_loader("apply", SourceFileLoader("apply", apply)) - apply = module_from_spec(spec) - spec.loader.exec_module(apply) - return apply +import sys +sys.path.append("../ansible") +import ansible # Utility for removing everything in a directory except one thing def remove_all_except_one(directory, exception): @@ -46,9 +37,8 @@ def create_workspace_dir(): # Create the workspace environment def create_workspace(env, playbook): workspace = create_workspace_dir() - apply = load_module() - apply.install_ansible(workspace / ".venv") - apply.create_workspace(workspace, env, playbook) + ansible.install_ansible(workspace / ".venv") + ansible.create_workspace(workspace, env, playbook) # Link the template into the workspace template_path = pathlib.Path(playbook).resolve() From 0f746836bb6a40bed03443e92df48014df7a9c31 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 15 Feb 2023 11:52:22 +0100 Subject: [PATCH 4/8] Properly handle not running shutdown --- ansible/roles/common/handlers/main.yml | 1 + ansible/roles/common/tasks/networking.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml index 2d9a3ddd5..92bfb1fec 100644 --- a/ansible/roles/common/handlers/main.yml +++ b/ansible/roles/common/handlers/main.yml @@ -26,3 +26,4 @@ - name: reboot reboot: + when: packer_build_name is undefined \ No newline at end of file diff --git a/ansible/roles/common/tasks/networking.yml b/ansible/roles/common/tasks/networking.yml index d701042ed..ca8901583 100644 --- a/ansible/roles/common/tasks/networking.yml +++ b/ansible/roles/common/tasks/networking.yml @@ -3,7 +3,7 @@ - name: set the system hostname hostname: name: "{{ inventory_hostname }}" - # notify: reboot + notify: reboot - name: upload /etc/hosts template: From 2af0d77b100f5898805878e3f486bee3893da7d3 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 15 Feb 2023 11:53:38 +0100 Subject: [PATCH 5/8] Pass the revision down from ssm --- ansible/roles/common/handlers/main.yml | 2 +- packer/docs-rs-builder/builder.pkr.hcl | 11 ++++++++++- packer/docs-rs-builder/variables.pkr.hcl | 4 ---- packer/packer | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) delete mode 100644 packer/docs-rs-builder/variables.pkr.hcl diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml index 92bfb1fec..821748dc3 100644 --- a/ansible/roles/common/handlers/main.yml +++ b/ansible/roles/common/handlers/main.yml @@ -26,4 +26,4 @@ - name: reboot reboot: - when: packer_build_name is undefined \ No newline at end of file + when: packer_build_name is undefined diff --git a/packer/docs-rs-builder/builder.pkr.hcl b/packer/docs-rs-builder/builder.pkr.hcl index c78146d21..99168e02e 100644 --- a/packer/docs-rs-builder/builder.pkr.hcl +++ b/packer/docs-rs-builder/builder.pkr.hcl @@ -7,8 +7,16 @@ packer { } } +data "amazon-parameterstore" "revision" { + name = "/docs-rs/builder/sha" +} + +locals { + revision = data.amazon-parameterstore.revision.value +} + source "amazon-ebs" "ubuntu" { - ami_name = "docs-rs-builder-${var.revision}" + ami_name = "docs-rs-builder-${local.revision}" instance_type = "t2.large" region = "us-east-1" source_ami_filter { @@ -41,5 +49,6 @@ build { playbook_file = "./play/playbook.yml" # The default is the user running packer user = "ubuntu" + extra_arguments = [ "--extra-vars", "vars_repository_sha=${local.revision}"] } } diff --git a/packer/docs-rs-builder/variables.pkr.hcl b/packer/docs-rs-builder/variables.pkr.hcl deleted file mode 100644 index 49928dd2c..000000000 --- a/packer/docs-rs-builder/variables.pkr.hcl +++ /dev/null @@ -1,4 +0,0 @@ -variable "revision" { - type = string - description = "the revision of the builder to build" -} \ No newline at end of file diff --git a/packer/packer b/packer/packer index b0864fc9d..d5a8ed127 100755 --- a/packer/packer +++ b/packer/packer @@ -51,7 +51,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("env") parser.add_argument("playbook") - parser.add_argument('--var', action='append', help='Ansible variables as $key=$value') + parser.add_argument('--var', action='append', help='Ansible variables as $key=$value', default=[]) args = parser.parse_args() workspace = create_workspace(args.env, args.playbook) From 177fad6b5f3b043deb665f1b7d4aae0499ddafdd Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 15 Feb 2023 16:45:13 +0100 Subject: [PATCH 6/8] Allow extra sudo users --- ansible/envs/staging/group_vars/docs-rs-builder.yml | 5 ++++- ansible/playbooks/docs-rs-builder.yml | 2 +- ansible/roles/common/files/ssh-keys/rylev.pub | 1 + packer/docs-rs-builder/builder.pkr.hcl | 9 +++++---- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ansible/envs/staging/group_vars/docs-rs-builder.yml b/ansible/envs/staging/group_vars/docs-rs-builder.yml index 9b8ab4581..7f1f07391 100644 --- a/ansible/envs/staging/group_vars/docs-rs-builder.yml +++ b/ansible/envs/staging/group_vars/docs-rs-builder.yml @@ -1,4 +1,7 @@ --- sha: "{{ lookup('aws_ssm', '/docs-rs/builder/sha') }}" -vars_repository_sha: "{{ sha | ternary(sha, 'HEAD') }}" \ No newline at end of file +vars_repository_sha: "{{ sha | ternary(sha, 'HEAD') }}" + +vars_extra_sudo_users: + - rylev diff --git a/ansible/playbooks/docs-rs-builder.yml b/ansible/playbooks/docs-rs-builder.yml index 1d84d1081..f0ea4b58a 100644 --- a/ansible/playbooks/docs-rs-builder.yml +++ b/ansible/playbooks/docs-rs-builder.yml @@ -9,6 +9,6 @@ - role: common papertrail_url: "{{ vars_papertrail_url }}" collect_metrics_from: "{{ global_collect_metrics_from }}" - sudo_users: "{{ global_sudo_users }}" + sudo_users: "{{ global_sudo_users + vars_extra_sudo_users }}" - role: docs-rs-builder diff --git a/ansible/roles/common/files/ssh-keys/rylev.pub b/ansible/roles/common/files/ssh-keys/rylev.pub index 1d1a2e531..835981d48 100644 --- a/ansible/roles/common/files/ssh-keys/rylev.pub +++ b/ansible/roles/common/files/ssh-keys/rylev.pub @@ -3,3 +3,4 @@ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDj2ga6r2r4AzZDyJJ3w81mTQIntuq5TdFlylZ1gwbd ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDro4UttZ6XQshnFBq4LdLXr0VDeunZFNRcv90ckpkZQrxbErzmpEvng0QUi9TI9gU/W4+cXhXAyEOcm+fVChesJBpqLFDJbQDmCPioPPsKVZ6ErBu3RHOgd4mD/Cfuly36L9AENql16R6ecxgbRVgpUISDYKo5jzRC7fJD40+bOai5Fv8+xvbuHPNJhj/IKxKCPDYCJ2+7H+6TMLZn5HjBvP42KES6030kR7pVWnug/OXSKESF0gm3tfspmFcw1aS57zVpyT7IlZvLb5zNvX8G8CEGV3KY//z80cbNMOa8QXggUGaLgFzwI2ng2W5CFHkOAXbX3bOtshsAmj0JWoM3ya+n70+E9tnGbwliOV/EfzQ1f0USyci8V2f1TkVLoRzWLBjtH5HsomsWN/8eNYcmDdbxy0TdclDEY6FavsDnQwD+JsDoeJaN+d31jyGSlYxcF+TZRgK5rFsRBZyXOc2sXi1bOQDWr5nt8y18yDhij7hj/wzV9DFwM4FVPOKNasImSZiVILwBUkdEGVAsTBuee6llrsApL1WmPsIb1xMrhN0+n1ZP7/07U69Eiqygbd6gb3a931H1z73j44MLvfh1BFsnrrmpFOb2PaSV+nwoi4pxSMOEiFZpno7hnc4OD3I+P/hBP3+6a42joHI42H08JsPZ0PgfiD8tjl/Z0/XLyw== ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILNkAIu1i0W8zWY/VO7yJ1I09KPlXa15Upfo8kIe21Up ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAHkoXhB9Pq+JKC+gPySI5yKwhYtGA++EfJ+7Ng3NNhN +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMxnE3/tgSLhXGQMjTzFBWBvpOJkNL+bojUthFVN4qCw diff --git a/packer/docs-rs-builder/builder.pkr.hcl b/packer/docs-rs-builder/builder.pkr.hcl index 99168e02e..07bafbca6 100644 --- a/packer/docs-rs-builder/builder.pkr.hcl +++ b/packer/docs-rs-builder/builder.pkr.hcl @@ -13,10 +13,12 @@ data "amazon-parameterstore" "revision" { locals { revision = data.amazon-parameterstore.revision.value + pretty_revision = substr(local.revision, 0, 8) + timestamp = regex_replace(timestamp(), "[- TZ:]", "") } source "amazon-ebs" "ubuntu" { - ami_name = "docs-rs-builder-${local.revision}" + ami_name = "docs-rs-builder-${local.pretty_revision}-${local.timestamp}" instance_type = "t2.large" region = "us-east-1" source_ami_filter { @@ -43,12 +45,11 @@ build { provisioner "ansible" { command = ".venv/bin/ansible-playbook" - # The default is "default" - host_alias = "docs-rs-builder" + groups = ["docs-rs-builder"] inventory_directory = "./env" playbook_file = "./play/playbook.yml" # The default is the user running packer user = "ubuntu" - extra_arguments = [ "--extra-vars", "vars_repository_sha=${local.revision}"] + extra_arguments = ["--extra-vars", "vars_repository_sha=${local.revision}"] } } From 0973037ec089b20767b481dfcfe3b23aee8cecd0 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Wed, 15 Feb 2023 16:58:43 +0100 Subject: [PATCH 7/8] Remove unused option for packer wrapper --- packer/README.md | 4 ++-- packer/packer | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packer/README.md b/packer/README.md index 3b995bf91..da58e113b 100644 --- a/packer/README.md +++ b/packer/README.md @@ -8,8 +8,8 @@ This directory contains a python script named `packer` which wraps the system `p ### Running `packer` -To run the wrapper pass the environment, playbook, and any vars ansible is expecting: +To run the wrapper pass the environment and playbook: ```bash -$ ./packer staging docs-rs-builder --var "revision=SOME_REVISION5 +$ ./packer staging docs-rs-build ``` diff --git a/packer/packer b/packer/packer index d5a8ed127..d3a7deaf6 100755 --- a/packer/packer +++ b/packer/packer @@ -51,15 +51,10 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("env") parser.add_argument("playbook") - parser.add_argument('--var', action='append', help='Ansible variables as $key=$value', default=[]) args = parser.parse_args() workspace = create_workspace(args.env, args.playbook) - # Call packer with the right arguments from the workspace - vars = [] - for var in args.var: - vars += ["-var", var] - cmd = ["packer", "build", *vars, "template"] + cmd = ["packer", "build", "template"] res = subprocess.run(cmd, cwd=str(workspace)) if res.returncode != 0: exit(1) From 95fd3112e92c962ae37a81818e1185eb710cfa21 Mon Sep 17 00:00:00 2001 From: Ryan Levick Date: Fri, 30 Jun 2023 15:14:36 +0200 Subject: [PATCH 8/8] Get working again --- ansible/{ansible.py => ansibleutils.py} | 0 ansible/apply | 8 ++++---- ansible/roles/common/tasks/apt.yml | 2 +- packer/README.md | 26 +++++++++++++++++++++++-- packer/docs-rs-builder/builder.pkr.hcl | 5 ++++- packer/packer | 8 +++++--- 6 files changed, 38 insertions(+), 11 deletions(-) rename ansible/{ansible.py => ansibleutils.py} (100%) diff --git a/ansible/ansible.py b/ansible/ansibleutils.py similarity index 100% rename from ansible/ansible.py rename to ansible/ansibleutils.py diff --git a/ansible/apply b/ansible/apply index 7ee256935..5d4e7ea8a 100755 --- a/ansible/apply +++ b/ansible/apply @@ -4,15 +4,15 @@ import pathlib import shutil import tempfile import argparse -import ansible +import ansibleutils def run_playbook(args): tempdir = pathlib.Path(tempfile.mkdtemp()) - ansible.create_workspace(tempdir, args.env, args.playbook) + ansibleutils.create_workspace(tempdir, args.env, args.playbook) try: # Invoke the ansible binary installed in the virtualenv ansible_args = [ - str(ansible.VENV_PATH / "bin" / "ansible-playbook"), + str(ansibleutils.VENV_PATH / "bin" / "ansible-playbook"), "-i", str(tempdir / "env" / "hosts"), str(tempdir / "play" / "playbook.yml"), ] @@ -39,5 +39,5 @@ if __name__ == "__main__": ) args = parser.parse_args() - ansible.install_ansible() + ansibleutils.install_ansible() run_playbook(args) diff --git a/ansible/roles/common/tasks/apt.yml b/ansible/roles/common/tasks/apt.yml index fd369e32b..e7b4d36eb 100644 --- a/ansible/roles/common/tasks/apt.yml +++ b/ansible/roles/common/tasks/apt.yml @@ -8,7 +8,7 @@ - name: install apt packages apt: name: - - aptitude # needed by ansible itself + # - aptitude # needed by ansible itself - ca-certificates - htop - iptables diff --git a/packer/README.md b/packer/README.md index da58e113b..3c512fc49 100644 --- a/packer/README.md +++ b/packer/README.md @@ -2,14 +2,36 @@ This directory contains configuration for running packer to build AMIs. +## Dependencies + +Running these packer scripts requires the following software: + +- python3 +- [packer](https://developer.hashicorp.com/packer/downloads) +- [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) + ## The `packer` wrapper This directory contains a python script named `packer` which wraps the system `packer`. The script creates an ansible virtual environment and moves the correct ansible configuration in place. ### Running `packer` -To run the wrapper pass the environment and playbook: +Before running the packer script, you will need to initialize packer: + +```bash +packer init ./docs-rs-builder +``` + +You will also need to make sure that you are logged into the correct AWS account in the AWS cli. First, ensure you have the configuration needed to log into the appropriate AWS account in your "~/.aws/config" file (TODO: link to detailed instructions). + +For example, to log into the docs-rs staging account, run: + +```bash +aws sso login --profile docs-rs-staging +``` + +To run the wrapper pass the environment and playbook along with the profile name of the aws account you just logged into: ```bash -$ ./packer staging docs-rs-build +$ AWS_PROFILE=docs-rs-staging ./packer staging docs-rs-builder ``` diff --git a/packer/docs-rs-builder/builder.pkr.hcl b/packer/docs-rs-builder/builder.pkr.hcl index 07bafbca6..871ac0e08 100644 --- a/packer/docs-rs-builder/builder.pkr.hcl +++ b/packer/docs-rs-builder/builder.pkr.hcl @@ -9,6 +9,7 @@ packer { data "amazon-parameterstore" "revision" { name = "/docs-rs/builder/sha" + region = "us-east-1" } locals { @@ -23,7 +24,7 @@ source "amazon-ebs" "ubuntu" { region = "us-east-1" source_ami_filter { filters = { - name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*" + name = "ubuntu/images/*ubuntu-jammy-22.04-amd64-server-*" root-device-type = "ebs" virtualization-type = "hvm" } @@ -51,5 +52,7 @@ build { # The default is the user running packer user = "ubuntu" extra_arguments = ["--extra-vars", "vars_repository_sha=${local.revision}"] + # Work around for https://github.com/hashicorp/packer-plugin-ansible/issues/69 + ansible_ssh_extra_args = ["-oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedKeyTypes=+ssh-rsa"] } } diff --git a/packer/packer b/packer/packer index d3a7deaf6..7b9399c14 100755 --- a/packer/packer +++ b/packer/packer @@ -6,8 +6,10 @@ import pathlib import shutil import argparse import sys +# This adds the ansible folder to the python package search path +# so we can reuse functionality from there. sys.path.append("../ansible") -import ansible +import ansibleutils # Utility for removing everything in a directory except one thing def remove_all_except_one(directory, exception): @@ -37,8 +39,8 @@ def create_workspace_dir(): # Create the workspace environment def create_workspace(env, playbook): workspace = create_workspace_dir() - ansible.install_ansible(workspace / ".venv") - ansible.create_workspace(workspace, env, playbook) + ansibleutils.install_ansible(workspace / ".venv") + ansibleutils.create_workspace(workspace, env, playbook) # Link the template into the workspace template_path = pathlib.Path(playbook).resolve()