diff --git a/.gitignore b/.gitignore index ed5c66cf6..90a232fc8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ /target /ansible/.venv /ansible/envs/dev +/packer/.workspace /terraform/shared/modules/lambda/packages /terraform/rds-databases/.forward-ports-cache-*.json /terragrunt/modules/aws-lambda/packages .terraform +packer-provisioner-* node_modules __pycache__ *.py[co] diff --git a/ansible/ansibleutils.py b/ansible/ansibleutils.py new file mode 100755 index 000000000..41a75e1f6 --- /dev/null +++ b/ansible/ansibleutils.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Utilities for creating the Ansible environment we use. + +import subprocess +import pathlib +import sys +import shutil + +BASE_PATH = pathlib.Path(__file__).resolve().parent +VENV_PATH = BASE_PATH / ".venv" + +# Ansible changes a lot between releases and deprecates a lot of stuff each of +# them. Using a pinned ansible identical between all team members should +# reduce the churn. +def install_ansible(venv_path = VENV_PATH): + requirements = BASE_PATH / "requirements.txt" + venv_requirements = venv_path / "installed-requirements.txt" + + # Avoid installing ansible in the virtualenv multiple times + if venv_requirements.exists() and \ + venv_requirements.read_bytes() == requirements.read_bytes(): + return + + print("creating a new virtual environment and install ansible in it...") + shutil.rmtree(venv_path, ignore_errors=True) + subprocess.run([sys.executable, "-m", "venv", str(venv_path)], check=True) + subprocess.run([ + str(venv_path / "bin" / "pip"), "install", "-r", str(requirements), + ], check=True) + shutil.copy(str(requirements), str(venv_requirements)) + +def create_workspace(dir, env, playbook): + env_dir = BASE_PATH / "envs" / env + # Create a temporary directory merging together the chosen + # environment, the chosen playbook and the shared files. + (dir / "play").mkdir() + (dir / "play" / "roles").symlink_to(BASE_PATH / "roles") + (dir / "play" / "group_vars").symlink_to(BASE_PATH / "group_vars") + (dir / "play" / "playbook.yml").symlink_to( + BASE_PATH / "playbooks" / (playbook + ".yml") + ) + (dir / "env").symlink_to(env_dir) + (dir / "ansible.cfg").symlink_to(BASE_PATH / "ansible.cfg") diff --git a/ansible/apply b/ansible/apply index f2147dd65..5d4e7ea8a 100755 --- a/ansible/apply +++ b/ansible/apply @@ -1,53 +1,18 @@ #!/usr/bin/env python3 import subprocess import pathlib -import sys import shutil import tempfile import argparse -import os - -BASE_PATH = pathlib.Path(__file__).resolve().parent -VENV_PATH = BASE_PATH / ".venv" - -# Ansible changes a lot between releases and deprecates a lot of stuff each of -# them. Using a pinned ansible identical between all team members should -# reduce the churn. -def install_ansible(): - requirements = BASE_PATH / "requirements.txt" - venv_requirements = VENV_PATH / "installed-requirements.txt" - - # Avoid installing ansible in the virtualenv multiple times - if venv_requirements.exists() and \ - venv_requirements.read_bytes() == requirements.read_bytes(): - return - - print("creating a new virtual environment and install ansible in it...") - shutil.rmtree(VENV_PATH, ignore_errors=True) - subprocess.run([sys.executable, "-m", "venv", str(VENV_PATH)], check=True) - subprocess.run([ - str(VENV_PATH / "bin" / "pip"), "install", "-r", str(requirements), - ], check=True) - shutil.copy(str(requirements), str(venv_requirements)) +import ansibleutils def run_playbook(args): - env_dir = BASE_PATH / "envs" / args.env tempdir = pathlib.Path(tempfile.mkdtemp()) + ansibleutils.create_workspace(tempdir, args.env, args.playbook) try: - # Create a temporary directory merging together the chosen - # environment, the chosen playbook and the shared files. - (tempdir / "play").mkdir() - (tempdir / "play" / "roles").symlink_to(BASE_PATH / "roles") - (tempdir / "play" / "group_vars").symlink_to(BASE_PATH / "group_vars") - (tempdir / "play" / "playbook.yml").symlink_to( - BASE_PATH / "playbooks" / (args.playbook + ".yml") - ) - (tempdir / "env").symlink_to(env_dir) - (tempdir / "ansible.cfg").symlink_to(BASE_PATH / "ansible.cfg") - - # Finally invoke the ansible binary installed in the virtualenv + # Invoke the ansible binary installed in the virtualenv ansible_args = [ - str(VENV_PATH / "bin" / "ansible-playbook"), + str(ansibleutils.VENV_PATH / "bin" / "ansible-playbook"), "-i", str(tempdir / "env" / "hosts"), str(tempdir / "play" / "playbook.yml"), ] @@ -74,5 +39,5 @@ if __name__ == "__main__": ) args = parser.parse_args() - install_ansible() + ansibleutils.install_ansible() run_playbook(args) diff --git a/ansible/envs/staging/group_vars/docs-rs-builder.yml b/ansible/envs/staging/group_vars/docs-rs-builder.yml index 9b8ab4581..7f1f07391 100644 --- a/ansible/envs/staging/group_vars/docs-rs-builder.yml +++ b/ansible/envs/staging/group_vars/docs-rs-builder.yml @@ -1,4 +1,7 @@ --- sha: "{{ lookup('aws_ssm', '/docs-rs/builder/sha') }}" -vars_repository_sha: "{{ sha | ternary(sha, 'HEAD') }}" \ No newline at end of file +vars_repository_sha: "{{ sha | ternary(sha, 'HEAD') }}" + +vars_extra_sudo_users: + - rylev diff --git a/ansible/playbooks/docs-rs-builder.yml b/ansible/playbooks/docs-rs-builder.yml index 1d84d1081..f0ea4b58a 100644 --- a/ansible/playbooks/docs-rs-builder.yml +++ b/ansible/playbooks/docs-rs-builder.yml @@ -9,6 +9,6 @@ - role: common papertrail_url: "{{ vars_papertrail_url }}" collect_metrics_from: "{{ global_collect_metrics_from }}" - sudo_users: "{{ global_sudo_users }}" + sudo_users: "{{ global_sudo_users + vars_extra_sudo_users }}" - role: docs-rs-builder diff --git a/ansible/roles/common/files/ssh-keys/rylev.pub b/ansible/roles/common/files/ssh-keys/rylev.pub index 1d1a2e531..835981d48 100644 --- a/ansible/roles/common/files/ssh-keys/rylev.pub +++ b/ansible/roles/common/files/ssh-keys/rylev.pub @@ -3,3 +3,4 @@ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDj2ga6r2r4AzZDyJJ3w81mTQIntuq5TdFlylZ1gwbd ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDro4UttZ6XQshnFBq4LdLXr0VDeunZFNRcv90ckpkZQrxbErzmpEvng0QUi9TI9gU/W4+cXhXAyEOcm+fVChesJBpqLFDJbQDmCPioPPsKVZ6ErBu3RHOgd4mD/Cfuly36L9AENql16R6ecxgbRVgpUISDYKo5jzRC7fJD40+bOai5Fv8+xvbuHPNJhj/IKxKCPDYCJ2+7H+6TMLZn5HjBvP42KES6030kR7pVWnug/OXSKESF0gm3tfspmFcw1aS57zVpyT7IlZvLb5zNvX8G8CEGV3KY//z80cbNMOa8QXggUGaLgFzwI2ng2W5CFHkOAXbX3bOtshsAmj0JWoM3ya+n70+E9tnGbwliOV/EfzQ1f0USyci8V2f1TkVLoRzWLBjtH5HsomsWN/8eNYcmDdbxy0TdclDEY6FavsDnQwD+JsDoeJaN+d31jyGSlYxcF+TZRgK5rFsRBZyXOc2sXi1bOQDWr5nt8y18yDhij7hj/wzV9DFwM4FVPOKNasImSZiVILwBUkdEGVAsTBuee6llrsApL1WmPsIb1xMrhN0+n1ZP7/07U69Eiqygbd6gb3a931H1z73j44MLvfh1BFsnrrmpFOb2PaSV+nwoi4pxSMOEiFZpno7hnc4OD3I+P/hBP3+6a42joHI42H08JsPZ0PgfiD8tjl/Z0/XLyw== ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILNkAIu1i0W8zWY/VO7yJ1I09KPlXa15Upfo8kIe21Up ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAHkoXhB9Pq+JKC+gPySI5yKwhYtGA++EfJ+7Ng3NNhN +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMxnE3/tgSLhXGQMjTzFBWBvpOJkNL+bojUthFVN4qCw diff --git a/ansible/roles/common/handlers/main.yml b/ansible/roles/common/handlers/main.yml index 2d9a3ddd5..821748dc3 100644 --- a/ansible/roles/common/handlers/main.yml +++ b/ansible/roles/common/handlers/main.yml @@ -26,3 +26,4 @@ - name: reboot reboot: + when: packer_build_name is undefined diff --git a/ansible/roles/common/tasks/apt.yml b/ansible/roles/common/tasks/apt.yml index fd369e32b..e7b4d36eb 100644 --- a/ansible/roles/common/tasks/apt.yml +++ b/ansible/roles/common/tasks/apt.yml @@ -8,7 +8,7 @@ - name: install apt packages apt: name: - - aptitude # needed by ansible itself + # - aptitude # needed by ansible itself - ca-certificates - htop - iptables diff --git a/packer/README.md b/packer/README.md new file mode 100644 index 000000000..3c512fc49 --- /dev/null +++ b/packer/README.md @@ -0,0 +1,37 @@ +# Packer + +This directory contains configuration for running packer to build AMIs. + +## Dependencies + +Running these packer scripts requires the following software: + +- python3 +- [packer](https://developer.hashicorp.com/packer/downloads) +- [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) + +## The `packer` wrapper + +This directory contains a python script named `packer` which wraps the system `packer`. The script creates an ansible virtual environment and moves the correct ansible configuration in place. + +### Running `packer` + +Before running the packer script, you will need to initialize packer: + +```bash +packer init ./docs-rs-builder +``` + +You will also need to make sure that you are logged into the correct AWS account in the AWS cli. First, ensure you have the configuration needed to log into the appropriate AWS account in your "~/.aws/config" file (TODO: link to detailed instructions). + +For example, to log into the docs-rs staging account, run: + +```bash +aws sso login --profile docs-rs-staging +``` + +To run the wrapper pass the environment and playbook along with the profile name of the aws account you just logged into: + +```bash +$ AWS_PROFILE=docs-rs-staging ./packer staging docs-rs-builder +``` diff --git a/packer/docs-rs-builder/builder.pkr.hcl b/packer/docs-rs-builder/builder.pkr.hcl new file mode 100644 index 000000000..871ac0e08 --- /dev/null +++ b/packer/docs-rs-builder/builder.pkr.hcl @@ -0,0 +1,58 @@ +packer { + required_plugins { + amazon = { + version = ">= 1.1.1" + source = "github.com/hashicorp/amazon" + } + } +} + +data "amazon-parameterstore" "revision" { + name = "/docs-rs/builder/sha" + region = "us-east-1" +} + +locals { + revision = data.amazon-parameterstore.revision.value + pretty_revision = substr(local.revision, 0, 8) + timestamp = regex_replace(timestamp(), "[- TZ:]", "") +} + +source "amazon-ebs" "ubuntu" { + ami_name = "docs-rs-builder-${local.pretty_revision}-${local.timestamp}" + instance_type = "t2.large" + region = "us-east-1" + source_ami_filter { + filters = { + name = "ubuntu/images/*ubuntu-jammy-22.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + } + ssh_username = "ubuntu" + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 64 + delete_on_termination = true + } +} + +build { + sources = [ + "source.amazon-ebs.ubuntu" + ] + + provisioner "ansible" { + command = ".venv/bin/ansible-playbook" + groups = ["docs-rs-builder"] + inventory_directory = "./env" + playbook_file = "./play/playbook.yml" + # The default is the user running packer + user = "ubuntu" + extra_arguments = ["--extra-vars", "vars_repository_sha=${local.revision}"] + # Work around for https://github.com/hashicorp/packer-plugin-ansible/issues/69 + ansible_ssh_extra_args = ["-oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedKeyTypes=+ssh-rsa"] + } +} diff --git a/packer/packer b/packer/packer new file mode 100755 index 000000000..7b9399c14 --- /dev/null +++ b/packer/packer @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import pathlib +import shutil +import argparse +import sys +# This adds the ansible folder to the python package search path +# so we can reuse functionality from there. +sys.path.append("../ansible") +import ansibleutils + +# Utility for removing everything in a directory except one thing +def remove_all_except_one(directory, exception): + for item in os.listdir(directory): + if item != exception: + item_path = os.path.join(directory, item) + if os.path.islink(item_path) or os.path.isfile(item_path): + # Delete the file or symlink + os.remove(item_path) + else: + # Delete the directory + shutil.rmtree(item_path) + +# Create the workspace directory leaving virtual environment +# if it's already there +def create_workspace_dir(): + dir = pathlib.Path(__file__).resolve().parent + workspace = os.path.join(dir, '.workspace') + if os.path.exists(workspace): + # Clean up workspace except for virtual environment + remove_all_except_one(workspace, ".venv") + else: + # Make workspace + os.mkdir(workspace) + return pathlib.Path(workspace) + +# Create the workspace environment +def create_workspace(env, playbook): + workspace = create_workspace_dir() + ansibleutils.install_ansible(workspace / ".venv") + ansibleutils.create_workspace(workspace, env, playbook) + + # Link the template into the workspace + template_path = pathlib.Path(playbook).resolve() + if not os.path.exists(template_path): + raise Exception(f"Last argument to packer call was the file '{template_path}' which does not exist") + (workspace / "template").symlink_to(template_path) + return workspace + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("env") + parser.add_argument("playbook") + args = parser.parse_args() + + workspace = create_workspace(args.env, args.playbook) + cmd = ["packer", "build", "template"] + res = subprocess.run(cmd, cwd=str(workspace)) + if res.returncode != 0: + exit(1)