From 997fb39223e03f2784a3cc9fba637d524f57ae2d Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Mon, 7 Aug 2023 19:22:15 +0200 Subject: [PATCH 01/26] init cleanup code Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 237 ++++++++++++++++++++++++++++++++++ plugins.pkr.hcl | 8 ++ sysbox-eks.pkr.hcl | 297 ++----------------------------------------- variables.pkr.hcl | 39 ++++++ 4 files changed, 297 insertions(+), 284 deletions(-) create mode 100644 build-ubuntu.pkr.hcl create mode 100644 plugins.pkr.hcl create mode 100644 variables.pkr.hcl diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl new file mode 100644 index 0000000..df02206 --- /dev/null +++ b/build-ubuntu.pkr.hcl @@ -0,0 +1,237 @@ +build { + name = "sysbox-eks" + sources = [ + "source.amazon-ebs.ubuntu-eks" + + ] + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo Updating apt", + "sudo apt-get update -y", + ] + } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + "export DEBIAN_FRONTEND=noninteractive", + + # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-sysbox + "echo '>>> Sysbox'", + "echo Downloading the Sysbox package", + "wget https://downloads.nestybox.com/sysbox/releases/v${var.sysbox_version}/sysbox-ce_${var.sysbox_version}-0.linux_amd64.deb", + + "echo Installing Sysbox package dependencies", + + "sudo apt-get install rsync -y", + + "echo Installing the Sysbox package", + "sudo dpkg --install ./sysbox-ce_*.linux_amd64.deb || true", # will fail due to missing dependencies, fixed in the next step + + "echo 'Fixing the Sysbox package (installing dependencies)'", + + "sudo --preserve-env=DEBIAN_FRONTEND apt-get install --fix-broken --yes --no-install-recommends", + + "echo Cleaning up", + "rm ./sysbox-ce_*.linux_amd64.deb", + ] + } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-shiftfs + "echo '>>> Shiftfs'", + + "echo Installing dependencies", + "sudo apt-get update", + "sudo apt-get install --yes --no-install-recommends make dkms git", + + "echo Detecting kernel version to determine the correct branch", + "export kernel_version=\"$(uname -r | sed --regexp-extended 's/([0-9]+\\.[0-9]+).*/\\1/g')\"", + "echo \"$kernel_version\"", + "declare -A kernel_to_branch=( [5.17]=k5.17 [5.16]=k5.16 [5.15]=k5.16 [5.14]=k5.13 [5.13]=k5.13 [5.10]=k5.10 [5.8]=k5.10 [5.4]=k5.4 )", + "export branch=\"$(echo $${kernel_to_branch[$kernel_version]})\"", + + "echo Cloning the repository branch: $branch", + "git clone --branch $branch --depth 1 --shallow-submodules https://github.com/toby63/shiftfs-dkms.git shiftfs", + "cd shiftfs", + + "echo Running the update script", + "./update1", + + "echo Building and installing", + "sudo make --file Makefile.dkms", + + "echo Cleaning up", + "cd ..", + "rm -rf shiftfs" + ] + } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + # https://github.com/cri-o/cri-o/blob/a68a72071e5004be78fe2b1b98cb3bfa0e51b74b/install.md#apt-based-operating-systems + "echo '>>> CRI-O'", + + # fixme(maximsmol): take into account ${ubuntu_version} + "export OS='xUbuntu_20.04'", + "export VERSION='${var.k8s_version}'", + + "echo Adding repositories", + "echo \"deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /\" | sudo dd status=none of=/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list", + "echo \"deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /\" | sudo dd status=none of=/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list", + + "echo Adding keys", + "mkdir --parents /usr/share/keyrings", + "curl --location https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$VERSION/$OS/Release.key | sudo gpg --dearmor --output /usr/share/keyrings/libcontainers-archive-keyring.gpg", + "curl --location https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo gpg --dearmor --output /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg", + + "echo Updating apt", + "sudo apt-get update", + + "echo Installing CRI-O", + "sudo apt-get install --yes --no-install-recommends cri-o cri-o-runc cri-tools", + + "echo Enabling CRI-O at startup", + "sudo systemctl enable crio" + ] + } + + + ## Uncomment this section to install from a patched CRI-O binary + # provisioner "file" { + # source = "crio" + # destination = "/home/ubuntu/crio" + # max_retries = 3 + # } + + # provisioner "shell" { + # inline = [ + # "echo >>> Installing prebuilt patched CRI-O", + # "sudo mv crio /usr/bin/crio", + # + # "echo Setting permissions", + # "sudo chmod u+x /usr/bin/crio" + # ] + # } + + ## Comment this section to install from a patched CRI-O binary + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Sysbox CRI-O patch'", + "echo Adding the Go backports repository", + "sudo apt-get install --yes --no-install-recommends software-properties-common", + "sudo add-apt-repository --yes ppa:longsleep/golang-backports", + + "echo Installing Go", + "sudo apt-get update", + # todo(maximsmol): lock the golang version + "sudo apt-get install --yes --no-install-recommends golang-go libgpgme-dev", + + "echo Cloning the patched CRI-O repository", + "git clone --branch v${var.k8s_version}-sysbox --depth 1 --shallow-submodules https://github.com/nestybox/cri-o.git cri-o", + + "echo Building", + "cd cri-o", + "make binaries", + + "echo Installing the patched binary", + "sudo mv bin/crio /usr/bin/crio", + "sudo chmod u+x /usr/bin/crio", + + + "echo Cleaning up", + "cd ..", + "rm -rf cri-o", + + "echo Restarting CRI-O", + "sudo systemctl restart crio" + ] + } + + provisioner "file" { + source = "bootstrap.sh.patch" + destination = "/home/ubuntu/bootstrap.sh.patch" + } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "sudo mv /home/ubuntu/bootstrap.sh.patch /usr/local/share/eks/bootstrap.sh.patch", + ] + } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + # Much of the rest of this is from inside the Sysbox K8s installer image + "echo '>>> Doing basic CRI-O configuration'", + + "echo Installing Dasel", + "sudo curl --location https://github.com/TomWright/dasel/releases/download/v1.24.3/dasel_linux_amd64 --output /usr/local/bin/dasel", + "sudo chmod u+x /usr/local/bin/dasel", + + # todo(maximsmol): do this only when K8s is configured without systemd cgroups (from sysbox todos) + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.cgroup_manager' 'cgroupfs'", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.conmon_cgroup' 'pod'", + # + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETFCAP", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple AUDIT_WRITE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple NET_RAW", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SYS_CHROOT", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple MKNOD", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple NET_BIND_SERVICE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple KILL", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple CHOWN", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETGID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETUID", + # + "sudo dasel put int --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.pids_limit' 16384", + # + "echo 'containers:231072:1048576' | sudo tee --append /etc/subuid", + "echo 'containers:231072:1048576' | sudo tee --append /etc/subgid", + # /usr/local/share/eks/bootstrap.sh is symlinked to /etc/eks/boostrap.sh + "sudo patch --backup /usr/local/share/eks/bootstrap.sh /usr/local/share/eks/bootstrap.sh.patch" + ] + } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Configuring CRI-O for Sysbox'", + + "echo Adding Sysbox to CRI-O runtimes", + "sudo dasel put object --parser toml --selector 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", + "sudo dasel put string --parser toml --selector 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", + ] + } + + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Removing /etc/cni/net.d'", + "sudo rm -r /etc/cni/net.d/", + ] + } +} diff --git a/plugins.pkr.hcl b/plugins.pkr.hcl new file mode 100644 index 0000000..f6a58da --- /dev/null +++ b/plugins.pkr.hcl @@ -0,0 +1,8 @@ +packer { + required_plugins { + amazon = { + version = "= 1.1.6" + source = "github.com/hashicorp/amazon" + } + } +} diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index 3c9106f..b4bda2b 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -1,45 +1,10 @@ -variable "ubuntu_version" { - type = string - default = "focal-20.04" - - validation { - condition = can(regex("^\\w+-\\d+\\.\\d+$", var.ubuntu_version)) - error_message = "Invalid Ubuntu version: expected '{name}-{major}.{minor}'." - } -} - -variable "sysbox_version" { - type = string - default = "0.6.2" - - validation { - condition = can(regex("^\\d+\\.\\d+\\.\\d+$", var.sysbox_version)) - error_message = "Invalid Sysbox version: expected '{major}.{minor}.{patch}'." - } -} - -variable "k8s_version" { - type = string - default = "1.23" - - validation { - condition = can(regex("^\\d+\\.\\d+$", var.k8s_version)) - error_message = "Invalid K8s version: expected '{major}.{minor}'." - } -} - -packer { - required_plugins { - amazon = { - version = "= 1.0.9" - source = "github.com/hashicorp/amazon" - } - } -} - source "amazon-ebs" "ubuntu-eks" { - ami_name = "latch-bio/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server" - ami_description = "Latch Bio, Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}, amd64 image" + ami_name = "${var.img_name}/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server" + ami_description = "Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}, amd64 image" + + region = "us-west-2" + instance_type = "t2.micro" + ami_regions = var.aws_target_regions tags = { Linux = "Ubuntu" @@ -58,250 +23,14 @@ source "amazon-ebs" "ubuntu-eks" { source_ami_filter { filters = { - name = "ubuntu-eks/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server-20230616" + name = "ubuntu-eks/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server-*" } - owners = ["099720109477"] - } - - region = "us-west-2" - instance_type = "t2.micro" - ssh_username = "ubuntu" -} - -build { - name = "sysbox-eks" - sources = [ - "source.amazon-ebs.ubuntu-eks" - - ] - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - "echo Updating apt", - "sudo apt-get update -y", - ] - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - "export DEBIAN_FRONTEND=noninteractive", - - # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-sysbox - "echo '>>> Sysbox'", - "echo Downloading the Sysbox package", - "wget https://downloads.nestybox.com/sysbox/releases/v${var.sysbox_version}/sysbox-ce_${var.sysbox_version}-0.linux_amd64.deb", - - "echo Installing Sysbox package dependencies", - - "sudo apt-get install rsync -y", - - "echo Installing the Sysbox package", - "sudo dpkg --install ./sysbox-ce_*.linux_amd64.deb || true", # will fail due to missing dependencies, fixed in the next step - - "echo 'Fixing the Sysbox package (installing dependencies)'", - - "sudo --preserve-env=DEBIAN_FRONTEND apt-get install --fix-broken --yes --no-install-recommends", - - "echo Cleaning up", - "rm ./sysbox-ce_*.linux_amd64.deb", - ] - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-shiftfs - "echo '>>> Shiftfs'", - - "echo Installing dependencies", - "sudo apt-get update", - "sudo apt-get install --yes --no-install-recommends make dkms git", - - "echo Detecting kernel version to determine the correct branch", - "export kernel_version=\"$(uname -r | sed --regexp-extended 's/([0-9]+\\.[0-9]+).*/\\1/g')\"", - "echo \"$kernel_version\"", - "declare -A kernel_to_branch=( [5.17]=k5.17 [5.16]=k5.16 [5.15]=k5.16 [5.14]=k5.13 [5.13]=k5.13 [5.10]=k5.10 [5.8]=k5.10 [5.4]=k5.4 )", - "export branch=\"$(echo $${kernel_to_branch[$kernel_version]})\"", - - "echo Cloning the repository branch: $branch", - "git clone --branch $branch --depth 1 --shallow-submodules https://github.com/toby63/shiftfs-dkms.git shiftfs", - "cd shiftfs", - - "echo Running the update script", - "./update1", - - "echo Building and installing", - "sudo make --file Makefile.dkms", - - "echo Cleaning up", - "cd ..", - "rm -rf shiftfs" - ] - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - # https://github.com/cri-o/cri-o/blob/a68a72071e5004be78fe2b1b98cb3bfa0e51b74b/install.md#apt-based-operating-systems - "echo '>>> CRI-O'", - - # fixme(maximsmol): take into account ${ubuntu_version} - "export OS='xUbuntu_20.04'", - "export VERSION='${var.k8s_version}'", - - "echo Adding repositories", - "echo \"deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /\" | sudo dd status=none of=/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list", - "echo \"deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /\" | sudo dd status=none of=/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list", - - "echo Adding keys", - "mkdir --parents /usr/share/keyrings", - "curl --location https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$VERSION/$OS/Release.key | sudo gpg --dearmor --output /usr/share/keyrings/libcontainers-archive-keyring.gpg", - "curl --location https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo gpg --dearmor --output /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg", - - "echo Updating apt", - "sudo apt-get update", - - "echo Installing CRI-O", - "sudo apt-get install --yes --no-install-recommends cri-o cri-o-runc cri-tools", - - "echo Enabling CRI-O at startup", - "sudo systemctl enable crio" - ] - } - - - ## Uncomment this section to install from a patched CRI-O binary - # provisioner "file" { - # source = "crio" - # destination = "/home/ubuntu/crio" - # max_retries = 3 - # } - - # provisioner "shell" { - # inline = [ - # "echo >>> Installing prebuilt patched CRI-O", - # "sudo mv crio /usr/bin/crio", - # - # "echo Setting permissions", - # "sudo chmod u+x /usr/bin/crio" - # ] - # } - - ## Comment this section to install from a patched CRI-O binary - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - - inline = [ - "set -o pipefail -o errexit", - - "echo '>>> Sysbox CRI-O patch'", - "echo Adding the Go backports repository", - "sudo apt-get install --yes --no-install-recommends software-properties-common", - "sudo add-apt-repository --yes ppa:longsleep/golang-backports", - - "echo Installing Go", - "sudo apt-get update", - # todo(maximsmol): lock the golang version - "sudo apt-get install --yes --no-install-recommends golang-go libgpgme-dev", - - "echo Cloning the patched CRI-O repository", - "git clone --branch v${var.k8s_version}-sysbox --depth 1 --shallow-submodules https://github.com/nestybox/cri-o.git cri-o", - - "echo Building", - "cd cri-o", - "make binaries", - - "echo Installing the patched binary", - "sudo mv bin/crio /usr/bin/crio", - "sudo chmod u+x /usr/bin/crio", - - - "echo Cleaning up", - "cd ..", - "rm -rf cri-o", - - "echo Restarting CRI-O", - "sudo systemctl restart crio" - ] + most_recent = true + owners = ["099720109477"] } - provisioner "file" { - source = "bootstrap.sh.patch" - destination = "/home/ubuntu/bootstrap.sh.patch" - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "sudo mv /home/ubuntu/bootstrap.sh.patch /usr/local/share/eks/bootstrap.sh.patch", - ] - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - # Much of the rest of this is from inside the Sysbox K8s installer image - "echo '>>> Doing basic CRI-O configuration'", - - "echo Installing Dasel", - "sudo curl --location https://github.com/TomWright/dasel/releases/download/v1.24.3/dasel_linux_amd64 --output /usr/local/bin/dasel", - "sudo chmod u+x /usr/local/bin/dasel", - - # todo(maximsmol): do this only when K8s is configured without systemd cgroups (from sysbox todos) - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.cgroup_manager' 'cgroupfs'", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.conmon_cgroup' 'pod'", - # - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETFCAP", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple AUDIT_WRITE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple NET_RAW", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SYS_CHROOT", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple MKNOD", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple NET_BIND_SERVICE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple KILL", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple CHOWN", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETGID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETUID", - # - "sudo dasel put int --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.pids_limit' 16384", - # - "echo 'containers:231072:1048576' | sudo tee --append /etc/subuid", - "echo 'containers:231072:1048576' | sudo tee --append /etc/subgid", - # /usr/local/share/eks/bootstrap.sh is symlinked to /etc/eks/boostrap.sh - "sudo patch --backup /usr/local/share/eks/bootstrap.sh /usr/local/share/eks/bootstrap.sh.patch" - ] - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - "echo '>>> Configuring CRI-O for Sysbox'", - - "echo Adding Sysbox to CRI-O runtimes", - "sudo dasel put object --parser toml --selector 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", - "sudo dasel put string --parser toml --selector 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", - ] - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - "echo '>>> Removing /etc/cni/net.d'", - "sudo rm -r /etc/cni/net.d/", - ] - } + ssh_username = "ubuntu" + # ami_groups = ["all"] # TODO: uncomment when ready to make public + force_deregister = true + force_delete_snapshot = true } diff --git a/variables.pkr.hcl b/variables.pkr.hcl new file mode 100644 index 0000000..5d09911 --- /dev/null +++ b/variables.pkr.hcl @@ -0,0 +1,39 @@ +variable "aws_target_regions" { + type = list(string) + default = ["us-east-1", "us-east-2", "us-west-2", "ap-southeast-2"] +} + +variable "img_name" { + type = string + default = "pluraldev" +} + +variable "ubuntu_version" { + type = string + default = "focal-20.04" + + validation { + condition = can(regex("^\\w+-\\d+\\.\\d+$", var.ubuntu_version)) + error_message = "Invalid Ubuntu version: expected '{name}-{major}.{minor}'." + } +} + +variable "sysbox_version" { + type = string + default = "0.6.2" + + validation { + condition = can(regex("^\\d+\\.\\d+\\.\\d+$", var.sysbox_version)) + error_message = "Invalid Sysbox version: expected '{major}.{minor}.{patch}'." + } +} + +variable "k8s_version" { + type = string + default = "1.23" + + validation { + condition = can(regex("^\\d+\\.\\d+$", var.k8s_version)) + error_message = "Invalid K8s version: expected '{major}.{minor}'." + } +} From 1dec3853fff8ce37173ecb91b70c487defc61201 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Mon, 7 Aug 2023 19:29:08 +0200 Subject: [PATCH 02/26] fix readme Signed-off-by: David van der Spek --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index f8049a1..2c8619a 100644 --- a/readme.md +++ b/readme.md @@ -8,7 +8,7 @@ Packer script for building an AMI with pre-installed Sysbox based on an Ubuntu E 1. Run `packer init .` 1. Ensure you have a patched CRI-O binary (for the Linux kernel 5.13) [The packer definition](./sysbox-eks.pkr.hcl) contains commented-out instructions on building the patched binary as part of the packer build. This is a very slow process so by default it copies the file instead. You could run the same commands on any Ubuntu 20.04 system and `scp` the binary to use it here. -1. Run `packer build sysbox-eks.pkr.hcl` +1. Run `packer build .` ## Differences from the Ubuntu EKS AMI From ffdb679b9e1fe4fe281a7133df6f435dd773043f Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Mon, 7 Aug 2023 21:54:00 +0200 Subject: [PATCH 03/26] fix patch and add bootstrap.sh for reference Signed-off-by: David van der Spek --- bootstrap.sh.patch | 49 ++--- build-ubuntu.pkr.hcl | 7 + current_bootstrap.sh | 511 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 541 insertions(+), 26 deletions(-) create mode 100644 current_bootstrap.sh diff --git a/bootstrap.sh.patch b/bootstrap.sh.patch index 4974b77..b5d98dc 100644 --- a/bootstrap.sh.patch +++ b/bootstrap.sh.patch @@ -1,5 +1,5 @@ ---- new_bootstrap.sh 2023-06-20 10:39:32.000000000 -0700 -+++ new_bootstrap.patched.sh 2023-06-20 10:42:53.000000000 -0700 +--- current_bootstrap.sh 2023-08-07 21:28:54 ++++ patched_bootstrap.sh 2023-08-07 21:29:11 @@ -146,7 +146,7 @@ API_RETRY_ATTEMPTS="${API_RETRY_ATTEMPTS:-3}" DOCKER_CONFIG_JSON="${DOCKER_CONFIG_JSON:-}" @@ -7,29 +7,26 @@ -DEFAULT_CONTAINER_RUNTIME="containerd" +DEFAULT_CONTAINER_RUNTIME="cri-o" CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-$DEFAULT_CONTAINER_RUNTIME}" - IP_FAMILY="${IP_FAMILY:-}" - SERVICE_IPV6_CIDR="${SERVICE_IPV6_CIDR:-}" -@@ -420,9 +420,21 @@ - systemctl restart docker - snap set kubelet-eks \ - container-runtime=docker + # from >= 1.27, the cloud-provider will be external + CLOUD_PROVIDER="aws" +@@ -429,6 +429,20 @@ + # see https://github.com/NVIDIA/k8s-device-plugin + cp /usr/local/share/eks/nvidia-runtime-config.toml /etc/containerd/config.toml + systemctl restart containerd ++ +elif [[ "$CONTAINER_RUNTIME" = "cri-o" ]]; then -+ echo "Container runtime is CRI-O" -+ snap set kubelet-eks \ -+ container-runtime=remote \ -+ container-runtime-endpoint=unix:///var/run/crio/crio.sock -+ dasel put \ -+ string \ -+ --parser toml \ -+ --file /etc/crio/crio.conf \ -+ --selector 'crio.image.pause_image' \ -+ "${PAUSE_CONTAINER}" -+ rm --force /run/dockershim.sock -+ ln -sf /run/crio/crio.sock /run/dockershim.sock ++ echo "Container runtime is CRI-O" ++ snap set kubelet-eks \ ++ container-runtime=remote \ ++ container-runtime-endpoint=unix:///var/run/crio/crio.sock ++ dasel put \ ++ string \ ++ --parser toml \ ++ --file /etc/crio/crio.conf \ ++ --selector 'crio.image.pause_image' \ ++ "${PAUSE_CONTAINER}" ++ rm --force /run/dockershim.sock ++ ln -sf /run/crio/crio.sock /run/dockershim.sock + else -- echo "Container runtime ${CONTAINER_RUNTIME} is not supported." -- exit 1 -+ echo "Custom container runtime." - fi - - echo "Configuring kubelet snap" + echo "Container runtime ${CONTAINER_RUNTIME} is not supported." diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index df02206..47762b7 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -5,6 +5,13 @@ build { ] + # Can be used to gen the current bootstrap.sh to update the patch +# provisioner "file" { +# source = "/usr/local/share/eks/bootstrap.sh" +# destination = "current_bootstrap.sh" +# direction = "download" +# } + provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ diff --git a/current_bootstrap.sh b/current_bootstrap.sh new file mode 100644 index 0000000..997a6e2 --- /dev/null +++ b/current_bootstrap.sh @@ -0,0 +1,511 @@ +#!/usr/bin/env bash +# CLOUD_IMG: This file was created/modified by the Cloud Image build process +# +# This file is part of the Ubuntu EKS image. This is a customized version of the +# Amazon bootstrap script for the use with Ubuntu EKS images. +# +# Copyright (C) 2020 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License version 3, as published by the +# Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program. If not, see . +# + +set -o pipefail +set -o nounset +set -o errexit + +err_report() { + echo "Exited with error on line $1" +} +trap 'err_report $LINENO' ERR + +IFS=$'\n\t' + +function print_help { + echo "usage: $0 [options] " + echo "Bootstraps an instance into an EKS cluster" + echo "" + echo "-h,--help print this help" + echo "--use-max-pods Sets --max-pods for the kubelet when true. (default: true)" + echo "--b64-cluster-ca The base64 encoded cluster CA content. Only valid when used with --apiserver-endpoint. Bypasses calling \"aws eks describe-cluster\"" + echo "--apiserver-endpoint The EKS cluster API Server endpoint. Only valid when used with --b64-cluster-ca. Bypasses calling \"aws eks describe-cluster\"" + echo "--kubelet-extra-args Extra arguments to add to the kubelet. Useful for adding labels or taints." + echo "--enable-docker-bridge Restores the docker default bridge network. (default: false)" + echo "--aws-api-retry-attempts Number of retry attempts for AWS API call (DescribeCluster) (default: 3)" + echo "--docker-config-json The contents of the /etc/docker/daemon.json file. Useful if you want a custom config differing from the default one in the AMI" + echo "--dns-cluster-ip Overrides the IP address to use for DNS queries within the cluster. Defaults to 10.100.0.10 or 172.20.0.10 based on the IP address of the primary interface" + echo "--pause-container-account The AWS account (number) to pull the pause container from" + echo "--pause-container-version The tag of the pause container" + echo "--container-runtime Specify a container runtime (default: containerd)" + echo "--ip-family Specify ip family of the cluster" + echo "--service-ipv6-cidr ipv6 cidr range of the cluster" +} + +POSITIONAL=() + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -h|--help) + print_help + exit 1 + ;; + --use-max-pods) + USE_MAX_PODS="$2" + shift + shift + ;; + --b64-cluster-ca) + B64_CLUSTER_CA=$2 + shift + shift + ;; + --apiserver-endpoint) + APISERVER_ENDPOINT=$2 + shift + shift + ;; + --kubelet-extra-args) + KUBELET_EXTRA_ARGS=$2 + shift + shift + ;; + --enable-docker-bridge) + ENABLE_DOCKER_BRIDGE=$2 + shift + shift + ;; + --aws-api-retry-attempts) + API_RETRY_ATTEMPTS=$2 + shift + shift + ;; + --docker-config-json) + DOCKER_CONFIG_JSON=$2 + shift + shift + ;; + --pause-container-account) + PAUSE_CONTAINER_ACCOUNT=$2 + shift + shift + ;; + --pause-container-version) + PAUSE_CONTAINER_VERSION=$2 + shift + shift + ;; + --dns-cluster-ip) + DNS_CLUSTER_IP=$2 + shift + shift + ;; + --container-runtime) + CONTAINER_RUNTIME=$2 + shift + shift + ;; + --ip-family) + IP_FAMILY=$2 + shift + shift + ;; + --service-ipv6-cidr) + SERVICE_IPV6_CIDR=$2 + shift + shift + ;; + *) # unknown option + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; + esac +done + +set +u +set -- "${POSITIONAL[@]}" # restore positional parameters +CLUSTER_NAME="$1" +set -u + +USE_MAX_PODS="${USE_MAX_PODS:-true}" +B64_CLUSTER_CA="${B64_CLUSTER_CA:-}" +APISERVER_ENDPOINT="${APISERVER_ENDPOINT:-}" +SERVICE_IPV4_CIDR="${SERVICE_IPV4_CIDR:-}" +DNS_CLUSTER_IP="${DNS_CLUSTER_IP:-}" +KUBELET_EXTRA_ARGS="${KUBELET_EXTRA_ARGS:-}" +ENABLE_DOCKER_BRIDGE="${ENABLE_DOCKER_BRIDGE:-false}" +API_RETRY_ATTEMPTS="${API_RETRY_ATTEMPTS:-3}" +DOCKER_CONFIG_JSON="${DOCKER_CONFIG_JSON:-}" +PAUSE_CONTAINER_VERSION="${PAUSE_CONTAINER_VERSION:-3.5}" +DEFAULT_CONTAINER_RUNTIME="containerd" +CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-$DEFAULT_CONTAINER_RUNTIME}" +# from >= 1.27, the cloud-provider will be external +CLOUD_PROVIDER="aws" +IP_FAMILY="${IP_FAMILY:-}" +SERVICE_IPV6_CIDR="${SERVICE_IPV6_CIDR:-}" + +echo "Using $CONTAINER_RUNTIME as the container runtime" + +# Helper function which calculates the amount of the given resource (either CPU or memory) +# to reserve in a given resource range, specified by a start and end of the range and a percentage +# of the resource to reserve. Note that we return zero if the start of the resource range is +# greater than the total resource capacity on the node. Additionally, if the end range exceeds the total +# resource capacity of the node, we use the total resource capacity as the end of the range. +# Args: +# $1 total available resource on the worker node in input unit (either millicores for CPU or Mi for memory) +# $2 start of the resource range in input unit +# $3 end of the resource range in input unit +# $4 percentage of range to reserve in percent*100 (to allow for two decimal digits) +# Return: +# amount of resource to reserve in input unit +get_resource_to_reserve_in_range() { + local total_resource_on_instance=$1 + local start_range=$2 + local end_range=$3 + local percentage=$4 + resources_to_reserve="0" + if (( $total_resource_on_instance > $start_range )); then + resources_to_reserve=$(((($total_resource_on_instance < $end_range ? \ + $total_resource_on_instance : $end_range) - $start_range) * $percentage / 100 / 100)) + fi + echo $resources_to_reserve +} + +# Calculates the amount of memory to reserve for kubeReserved in mebibytes. KubeReserved is a function of pod +# density so we are calculating the amount of memory to reserve for Kubernetes systems daemons by +# considering the maximum number of pods this instance type supports. +# Args: +# $1 the max number of pods per instance type (MAX_PODS) based on values from /etc/eks/eni-max-pods.txt +# Return: +# memory to reserve in Mi for the kubelet +get_memory_mebibytes_to_reserve() { + local max_num_pods=$1 + memory_to_reserve=$((11 * $max_num_pods + 255)) + echo $memory_to_reserve +} + +# Calculates the amount of CPU to reserve for kubeReserved in millicores from the total number of vCPUs available on the instance. +# From the total core capacity of this worker node, we calculate the CPU resources to reserve by reserving a percentage +# of the available cores in each range up to the total number of cores available on the instance. +# We are using these CPU ranges from GKE (https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture#node_allocatable): +# 6% of the first core +# 1% of the next core (up to 2 cores) +# 0.5% of the next 2 cores (up to 4 cores) +# 0.25% of any cores above 4 cores +# Return: +# CPU resources to reserve in millicores (m) +get_cpu_millicores_to_reserve() { + local total_cpu_on_instance=$(($(nproc) * 1000)) + local cpu_ranges=(0 1000 2000 4000 $total_cpu_on_instance) + local cpu_percentage_reserved_for_ranges=(600 100 50 25) + cpu_to_reserve="0" + for i in ${!cpu_percentage_reserved_for_ranges[@]}; do + local start_range=${cpu_ranges[$i]} + local end_range=${cpu_ranges[(($i+1))]} + local percentage_to_reserve_for_range=${cpu_percentage_reserved_for_ranges[$i]} + cpu_to_reserve=$(($cpu_to_reserve + \ + $(get_resource_to_reserve_in_range $total_cpu_on_instance $start_range $end_range $percentage_to_reserve_for_range))) + done + echo $cpu_to_reserve +} + +if [ -z "$CLUSTER_NAME" ]; then + echo "CLUSTER_NAME is not defined" + exit 1 +fi + +if [[ ! -z "${IP_FAMILY}" ]]; then + if [[ "${IP_FAMILY}" != "ipv4" ]] && [[ "${IP_FAMILY}" != "ipv6" ]] ; then + echo "Invalid IpFamily. Only ipv4 or ipv6 are allowed" + exit 1 + fi + + if [[ "${IP_FAMILY}" == "ipv6" ]] && [[ ! -z "${B64_CLUSTER_CA}" ]] && [[ ! -z "${APISERVER_ENDPOINT}" ]] && [[ -z "${SERVICE_IPV6_CIDR}" ]]; then + echo "Service Ipv6 Cidr must be provided when ip-family is specified as IPV6" + exit 1 + fi +fi + +if [[ ! -z "${SERVICE_IPV6_CIDR}" ]]; then + if [[ "${IP_FAMILY}" == "ipv4" ]]; then + echo "ip-family should be ipv6 when service-ipv6-cidr is specified" + exit 1 + fi + IP_FAMILY="ipv6" +fi + +echo "Aliasing EKS k8s snap commands" +snap alias kubelet-eks.kubelet kubelet +snap alias kubectl-eks.kubectl kubectl + +echo "Stopping k8s daemons until configured" +snap stop kubelet-eks +# Flush the restart-rate for failed starts + +AWS_DEFAULT_REGION=$(/usr/local/share/eks/imds 'latest/dynamic/instance-identity/document' | jq .region -r) +AWS_SERVICES_DOMAIN=$(/usr/local/share/eks/imds '2018-09-24/meta-data/services/domain') + +MACHINE=$(uname -m) +if [[ "$MACHINE" != "x86_64" && "$MACHINE" != "aarch64" ]]; then + echo "Unknown machine architecture '$MACHINE'" >&2 + exit 1 +fi + +ECR_URI=$(/etc/eks/get-ecr-uri.sh "${AWS_DEFAULT_REGION}" "${AWS_SERVICES_DOMAIN}" "${PAUSE_CONTAINER_ACCOUNT:-}") +PAUSE_CONTAINER_IMAGE=${PAUSE_CONTAINER_IMAGE:-$ECR_URI/eks/pause} +PAUSE_CONTAINER="$PAUSE_CONTAINER_IMAGE:$PAUSE_CONTAINER_VERSION" + +### kubelet kubeconfig + +CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki +CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt +mkdir -p $CA_CERTIFICATE_DIRECTORY +if [[ -z "${B64_CLUSTER_CA}" ]] || [[ -z "${APISERVER_ENDPOINT}" ]]; then + DESCRIBE_CLUSTER_RESULT="/tmp/describe_cluster_result.txt" + + # Retry the DescribeCluster API for API_RETRY_ATTEMPTS + for attempt in `seq 0 $API_RETRY_ATTEMPTS`; do + rc=0 + if [[ $attempt -gt 0 ]]; then + echo "Attempt $attempt of $API_RETRY_ATTEMPTS" + fi + + aws eks wait cluster-active \ + --region=${AWS_DEFAULT_REGION} \ + --name=${CLUSTER_NAME} + + aws eks describe-cluster \ + --region=${AWS_DEFAULT_REGION} \ + --name=${CLUSTER_NAME} \ + --output=text \ + --query 'cluster.{certificateAuthorityData: certificateAuthority.data, endpoint: endpoint, serviceIpv4Cidr: kubernetesNetworkConfig.serviceIpv4Cidr, serviceIpv6Cidr: kubernetesNetworkConfig.serviceIpv6Cidr, clusterIpFamily: kubernetesNetworkConfig.ipFamily}' > $DESCRIBE_CLUSTER_RESULT || rc=$? + if [[ $rc -eq 0 ]]; then + break + fi + if [[ $attempt -eq $API_RETRY_ATTEMPTS ]]; then + exit $rc + fi + jitter=$((1 + RANDOM % 10)) + sleep_sec="$(( $(( 5 << $((1+$attempt)) )) + $jitter))" + sleep $sleep_sec + done + B64_CLUSTER_CA=$(cat $DESCRIBE_CLUSTER_RESULT | awk '{print $1}') + APISERVER_ENDPOINT=$(cat $DESCRIBE_CLUSTER_RESULT | awk '{print $3}') + SERVICE_IPV4_CIDR=$(cat $DESCRIBE_CLUSTER_RESULT | awk '{print $4}') + SERVICE_IPV6_CIDR=$(cat $DESCRIBE_CLUSTER_RESULT | awk '{print $5}') + + if [[ -z "${IP_FAMILY}" ]]; then + IP_FAMILY=$(cat $DESCRIBE_CLUSTER_RESULT | awk '{print $2}') + fi +fi + +if [[ -z "${IP_FAMILY}" ]] || [[ "${IP_FAMILY}" == "None" ]]; then + ### this can happen when the ifFamily field is not found in describeCluster response + ### or B64_CLUSTER_CA and APISERVER_ENDPOINT are defined but IPFamily isn't + IP_FAMILY="ipv4" +fi + +echo $B64_CLUSTER_CA | base64 -d > $CA_CERTIFICATE_FILE_PATH + +sed -i s,CLUSTER_NAME,$CLUSTER_NAME,g /var/lib/kubelet/kubeconfig +sed -i s,MASTER_ENDPOINT,$APISERVER_ENDPOINT,g /var/lib/kubelet/kubeconfig +sed -i s,AWS_REGION,$AWS_DEFAULT_REGION,g /var/lib/kubelet/kubeconfig +/snap/bin/kubectl config \ + --kubeconfig /var/lib/kubelet/kubeconfig \ + set-cluster \ + kubernetes \ + --certificate-authority=/etc/kubernetes/pki/ca.crt \ + --server=$APISERVER_ENDPOINT + +### kubelet.service configuration + +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + DNS_CLUSTER_IP=$(awk -F/ '{print $1}' <<< $SERVICE_IPV6_CIDR)a +fi + +MAC=$(/usr/local/share/eks/imds 'latest/meta-data/mac') + +if [[ -z "${DNS_CLUSTER_IP}" ]]; then + if [[ ! -z "${SERVICE_IPV4_CIDR}" ]] && [[ "${SERVICE_IPV4_CIDR}" != "None" ]] ; then + #Sets the DNS Cluster IP address that would be chosen from the serviceIpv4Cidr. (x.y.z.10) + DNS_CLUSTER_IP=${SERVICE_IPV4_CIDR%.*}.10 + else + TEN_RANGE=$(/usr/local/share/eks/imds "latest/meta-data/network/interfaces/macs/$MAC/vpc-ipv4-cidr-blocks" | grep -c '^10\..*' || true ) + DNS_CLUSTER_IP=10.100.0.10 + if [[ "$TEN_RANGE" != "0" ]]; then + DNS_CLUSTER_IP=172.20.0.10 + fi + fi +else + DNS_CLUSTER_IP="${DNS_CLUSTER_IP}" +fi + +KUBELET_CONFIG=/etc/kubernetes/kubelet/kubelet-config.json +snap set kubelet-eks cluster-dns="$DNS_CLUSTER_IP" + +if [[ "${IP_FAMILY}" == "ipv4" ]]; then + INTERNAL_IP=$(/usr/local/share/eks/imds 'latest/meta-data/local-ipv4') +else + INTERNAL_IP_URI=latest/meta-data/network/interfaces/macs/$MAC/ipv6s + INTERNAL_IP=$(/usr/local/share/eks/imds $INTERNAL_IP_URI) +fi +INSTANCE_TYPE=$(/usr/local/share/eks/imds 'latest/meta-data/instance-type') + +# Sets kubeReserved and evictionHard in /etc/kubernetes/kubelet/kubelet-config.json for worker nodes. The following two function +# calls calculate the CPU and memory resources to reserve for kubeReserved based on the instance type of the worker node. +# Note that allocatable memory and CPU resources on worker nodes is calculated by the Kubernetes scheduler +# with this formula when scheduling pods: Allocatable = Capacity - Reserved - Eviction Threshold. + +#calculate the max number of pods per instance type +MAX_PODS_FILE="/etc/eks/eni-max-pods.txt" +set +o pipefail +MAX_PODS=$(cat $MAX_PODS_FILE | awk "/^${INSTANCE_TYPE:-unset}/"' { print $2 }') +set -o pipefail +if [ -z "$MAX_PODS" ] || [ -z "$INSTANCE_TYPE" ]; then + log "INFO: No entry for type '$INSTANCE_TYPE' in $MAX_PODS_FILE. Will attempt to auto-discover value." + # When determining the value of maxPods, we're using the legacy calculation by default since it's more restrictive than + # the PrefixDelegation based alternative and is likely to be in-use by more customers. + # The legacy numbers also maintain backwards compatibility when used to calculate `kubeReserved.memory` + MAX_PODS=$(/etc/eks/max-pods-calculator.sh --instance-type-from-imds --cni-version 1.10.0 --show-max-allowed) +fi + +# calculates the amount of each resource to reserve +mebibytes_to_reserve=$(get_memory_mebibytes_to_reserve $MAX_PODS) +cpu_millicores_to_reserve=$(get_cpu_millicores_to_reserve) +# writes kubeReserved and evictionHard to the kubelet-config using the amount of CPU and memory to be reserved +echo "$(jq '. += {"evictionHard": {"memory.available": "100Mi", "nodefs.available": "10%", "nodefs.inodesFree": "5%"}}' $KUBELET_CONFIG)" > $KUBELET_CONFIG +echo "$(jq --arg mebibytes_to_reserve "${mebibytes_to_reserve}Mi" --arg cpu_millicores_to_reserve "${cpu_millicores_to_reserve}m" \ + '. += {kubeReserved: {"cpu": $cpu_millicores_to_reserve, "ephemeral-storage": "1Gi", "memory": $mebibytes_to_reserve}}' $KUBELET_CONFIG)" > $KUBELET_CONFIG + +if [[ "$USE_MAX_PODS" = "true" ]]; then + echo "$(jq ".maxPods=$MAX_PODS" $KUBELET_CONFIG)" > $KUBELET_CONFIG +fi + +if [[ "$CONTAINER_RUNTIME" = "containerd" ]]; then + echo "Container runtime is containerd" + mkdir -p /etc/systemd/system/containerd.service.d + # Symlink is needed for pull-sandbox-image.sh + cat < /etc/systemd/system/containerd.service.d/10-compat-symlink.conf +[Service] +ExecStartPre=/bin/ln -sf /run/containerd/containerd.sock /run/dockershim.sock +EOF + systemctl daemon-reload + sed "s,SANDBOX_IMAGE,$PAUSE_CONTAINER,g" \ + /etc/containerd/config.toml + systemctl restart containerd + /usr/local/share/eks/pull-sandbox-image.sh + snap set kubelet-eks \ + container-runtime=remote \ + container-runtime-endpoint=unix:///run/containerd/containerd.sock + +elif [[ "$CONTAINER_RUNTIME" = "dockerd" ]]; then + echo "Container runtime is docker" + mkdir -p /etc/docker + if [[ -n "$DOCKER_CONFIG_JSON" ]]; then + echo "$DOCKER_CONFIG_JSON" > /etc/docker/daemon.json + fi + if [[ "$ENABLE_DOCKER_BRIDGE" = "true" ]]; then + # Enabling the docker bridge network. We have to disable live-restore as it + # prevents docker from recreating the default bridge network on restart + echo "$(jq '.bridge="docker0" | ."live-restore"=false' /etc/docker/daemon.json)" > /etc/docker/daemon.json + fi + systemctl restart docker + snap set kubelet-eks \ + container-runtime=docker + +elif [[ "$CONTAINER_RUNTIME" = "nvidia-container-runtime" ]]; then + echo "Container runtime is ${CONTAINER_RUNTIME}" + # update config.toml file + # see https://github.com/NVIDIA/k8s-device-plugin + cp /usr/local/share/eks/nvidia-runtime-config.toml /etc/containerd/config.toml + systemctl restart containerd + +else + echo "Container runtime ${CONTAINER_RUNTIME} is not supported." + exit 1 +fi + +if [[ "$CLOUD_PROVIDER" = "external" ]]; then + echo "cloud-provider is $CLOUD_PROVIDER" + # When the external cloud provider is used, kubelet will use /etc/hostname as the name of the Node object. + # If the VPC has a custom `domain-name` in its DHCP options set, and the VPC has `enableDnsHostnames` set to `true`, + # then /etc/hostname is not the same as EC2's PrivateDnsName. + # The name of the Node object must be equal to EC2's PrivateDnsName for the aws-iam-authenticator to allow this kubelet to manage it. + INSTANCE_ID=$(/usr/local/share/eks/imds /latest/meta-data/instance-id) + REGION=$(/usr/local/share/eks/imds /latest/meta-data/placement/region) + PRIVATE_DNS_NAME=$(AWS_RETRY_MODE=standard AWS_MAX_ATTEMPTS=10 aws ec2 describe-instances --region $REGION --instance-ids $INSTANCE_ID --query 'Reservations[].Instances[].PrivateDnsName' --output text) + + snap set kubelet-eks \ + hostname-override=$PRIVATE_DNS_NAME \ + image-credential-provider-config=/etc/eks/ecr-credential-provider/config.json \ + image-credential-provider-bin-dir=/etc/eks/ecr-credential-provider +fi + +# gpu boost clock +if command -v nvidia-smi &>/dev/null && test "$CONTAINER_RUNTIME" = "nvidia-container-runtime"; then + echo "nvidia-smi found" + + nvidia-smi -q > /tmp/nvidia-smi-check + if [[ "$?" == "0" ]]; then + sudo nvidia-smi -pm 1 # set persistence mode + sudo nvidia-smi --auto-boost-default=0 + + GPUNAME=$(nvidia-smi -L | head -n1) + echo $GPUNAME + + # set application clock to maximum + if [[ $GPUNAME == *"A100"* ]]; then + nvidia-smi -ac 1215,1410 + elif [[ $GPUNAME == *"V100"* ]]; then + nvidia-smi -ac 877,1530 + elif [[ $GPUNAME == *"K80"* ]]; then + nvidia-smi -ac 2505,875 + elif [[ $GPUNAME == *"T4"* ]]; then + nvidia-smi -ac 5001,1590 + elif [[ $GPUNAME == *"M60"* ]]; then + nvidia-smi -ac 2505,1177 + else + echo "unsupported gpu" + fi + else + cat /tmp/nvidia-smi-check + fi +else + echo "nvidia-smi not found" +fi + +echo "Configuring kubelet snap" +snap set kubelet-eks \ + address=0.0.0.0 \ + anonymous-auth=false \ + authentication-token-webhook=true \ + authorization-mode=Webhook \ + cgroup-driver=cgroupfs \ + client-ca-file="$CA_CERTIFICATE_FILE_PATH" \ + cloud-provider="$CLOUD_PROVIDER" \ + cluster-domain=cluster.local \ + cni-bin-dir=/opt/cni/bin \ + cni-conf-dir=/etc/cni/net.d \ + config="$KUBELET_CONFIG" \ + feature-gates=RotateKubeletServerCertificate=true \ + kubeconfig=/var/lib/kubelet/kubeconfig \ + node-ip="$INTERNAL_IP" \ + network-plugin=cni \ + register-node=true \ + resolv-conf=/run/systemd/resolve/resolv.conf \ + pod-infra-container-image="$PAUSE_CONTAINER" + +snap set kubelet-eks args="$KUBELET_EXTRA_ARGS" + +echo "Starting k8s kubelet daemon" +snap start --enable kubelet-eks From 90e0016f4fecd94b63caa47492841e206c446350 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Tue, 8 Aug 2023 16:02:41 +0200 Subject: [PATCH 04/26] some changes compared to upstream Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 45 +++++++++++++++++++++++++++++--------------- variables.pkr.hcl | 2 +- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index 47762b7..0507f6a 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -195,22 +195,37 @@ build { "sudo curl --location https://github.com/TomWright/dasel/releases/download/v1.24.3/dasel_linux_amd64 --output /usr/local/bin/dasel", "sudo chmod u+x /usr/local/bin/dasel", - # todo(maximsmol): do this only when K8s is configured without systemd cgroups (from sysbox todos) - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.cgroup_manager' 'cgroupfs'", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.conmon_cgroup' 'pod'", + # Disable selinux for now. + "sudo dasel put bool --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.selinux' false", + + # overlayfs with metacopy=on improves startup time of CRI-O rootless containers significantly + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.storage_driver' 'overlay'", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.storage_option.[]' 'overlay.mountopt=metacopy=on'", + # - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETFCAP", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple AUDIT_WRITE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple NET_RAW", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SYS_CHROOT", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple MKNOD", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple NET_BIND_SERVICE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple KILL", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple CHOWN", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETGID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.default_capabilities.[]' --multiple SETUID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' CHOWN", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' DAC_OVERRIDE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' FSETID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' FOWNER", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETUID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETGID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETPCAP", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETFCAP", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' NET_BIND_SERVICE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' KILL", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' AUDIT_WRITE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' NET_RAW", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SYS_CHROOT", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' MKNOD", # "sudo dasel put int --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.pids_limit' 16384", + + # Create 'crio.image' table (required for 'pause_image' settings). + "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.image'", + + # Create 'crio.network' table (required for 'network_dir' settings). + "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.network'", + # "echo 'containers:231072:1048576' | sudo tee --append /etc/subuid", "echo 'containers:231072:1048576' | sudo tee --append /etc/subgid", @@ -227,8 +242,8 @@ build { "echo '>>> Configuring CRI-O for Sysbox'", "echo Adding Sysbox to CRI-O runtimes", - "sudo dasel put object --parser toml --selector 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", - "sudo dasel put string --parser toml --selector 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", + "sudo dasel put object --parser toml --selector -m 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", + "sudo dasel put string --parser toml --selector -m 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", ] } diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 5d09911..dc4b4cb 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev" + default = "pluraldev-02" } variable "ubuntu_version" { From 0d84d866a7908b7849ffff4436db928cd5fc8a13 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Tue, 8 Aug 2023 16:20:43 +0200 Subject: [PATCH 05/26] fix indent issue Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index 0507f6a..7b88887 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -196,11 +196,11 @@ build { "sudo chmod u+x /usr/local/bin/dasel", # Disable selinux for now. - "sudo dasel put bool --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.selinux' false", + "sudo dasel put bool --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.selinux' false", # overlayfs with metacopy=on improves startup time of CRI-O rootless containers significantly "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.storage_driver' 'overlay'", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.storage_option.[]' 'overlay.mountopt=metacopy=on'", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.storage_option.[]' 'overlay.mountopt=metacopy=on'", # "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' CHOWN", @@ -221,10 +221,10 @@ build { "sudo dasel put int --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.pids_limit' 16384", # Create 'crio.image' table (required for 'pause_image' settings). - "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.image'", + "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.image'", - # Create 'crio.network' table (required for 'network_dir' settings). - "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.network'", + # Create 'crio.network' table (required for 'network_dir' settings). + "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.network'", # "echo 'containers:231072:1048576' | sudo tee --append /etc/subuid", From 05ec54a4f02abd124c1366a2e619879e3ec19fba Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Tue, 8 Aug 2023 16:51:24 +0200 Subject: [PATCH 06/26] remove --selector from dasel commands Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index 7b88887..0155f44 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -196,35 +196,35 @@ build { "sudo chmod u+x /usr/local/bin/dasel", # Disable selinux for now. - "sudo dasel put bool --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.selinux' false", + "sudo dasel put bool --parser toml --file /etc/crio/crio.conf 'crio.runtime.selinux' false", # overlayfs with metacopy=on improves startup time of CRI-O rootless containers significantly - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector 'crio.storage_driver' 'overlay'", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.storage_option.[]' 'overlay.mountopt=metacopy=on'", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf 'crio.storage_driver' 'overlay'", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.storage_option.[]' 'overlay.mountopt=metacopy=on'", # - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' CHOWN", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' DAC_OVERRIDE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' FSETID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' FOWNER", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETUID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETGID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETPCAP", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SETFCAP", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' NET_BIND_SERVICE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' KILL", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' AUDIT_WRITE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' NET_RAW", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' SYS_CHROOT", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf --selector -m 'crio.runtime.default_capabilities.[]' MKNOD", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' CHOWN", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' DAC_OVERRIDE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' FSETID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' FOWNER", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETUID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETGID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETPCAP", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETFCAP", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' NET_BIND_SERVICE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' KILL", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' AUDIT_WRITE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' NET_RAW", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SYS_CHROOT", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' MKNOD", # - "sudo dasel put int --parser toml --file /etc/crio/crio.conf --selector 'crio.runtime.pids_limit' 16384", + "sudo dasel put int --parser toml --file /etc/crio/crio.conf 'crio.runtime.pids_limit' 16384", # Create 'crio.image' table (required for 'pause_image' settings). - "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.image'", + "sudo dasel put document --parser toml --file /etc/crio/crio.conf '.crio.image'", # Create 'crio.network' table (required for 'network_dir' settings). - "sudo dasel put document --parser toml --file /etc/crio/crio.conf --selector '.crio.network'", + "sudo dasel put document --parser toml --file /etc/crio/crio.conf '.crio.network'", # "echo 'containers:231072:1048576' | sudo tee --append /etc/subuid", @@ -242,8 +242,8 @@ build { "echo '>>> Configuring CRI-O for Sysbox'", "echo Adding Sysbox to CRI-O runtimes", - "sudo dasel put object --parser toml --selector -m 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", - "sudo dasel put string --parser toml --selector -m 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", + "sudo dasel put object --parser toml -m 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", + "sudo dasel put string --parser toml -m 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", ] } From 73cbd847f4a070e4fee19c0c1636a5fafefef184 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Tue, 8 Aug 2023 17:44:18 +0200 Subject: [PATCH 07/26] Add makefile to download patched cri-o binaries Signed-off-by: David van der Spek --- Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3fc9b34 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +SYSBOX_VERSION=v0.6.2 + +get-crio: + rm -rf ./crio + docker run --rm -it --platform linux/amd64 -v ./crio/amd64:/host/crio registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp -r /opt/crio-deploy/bin/* /host/crio/" + docker run --rm -it --platform linux/arm64 -v ./crio/arm64:/host/crio registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp -r /opt/crio-deploy/bin/* /host/crio/" +# remove tar.gz files + find ./crio/ -path '**/*.tar.gz' -delete +# remove empty directories + find ./crio/ -empty -type d -delete From f22357b46801fecd1a867f8576270ea491905003 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Tue, 8 Aug 2023 18:31:15 +0200 Subject: [PATCH 08/26] try fixing crio config Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 11 +++++++++++ variables.pkr.hcl | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index 0155f44..ab44364 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -202,6 +202,12 @@ build { "sudo dasel put string --parser toml --file /etc/crio/crio.conf 'crio.storage_driver' 'overlay'", "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.storage_option.[]' 'overlay.mountopt=metacopy=on'", + # todo(maximsmol): do this only when K8s is configured without systemd cgroups (from sysbox todos) + # this is done by the kubelet-config-helper.sh + # see https://github.com/nestybox/sysbox-pkgr/blob/b560194d516b300e9e201274a29348d3626055c1/k8s/scripts/kubelet-config-helper.sh#L861 + "sudo dasel put string --parser toml --file /etc/crio/crio.conf 'crio.runtime.cgroup_manager' 'cgroupfs'", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf 'crio.runtime.conmon_cgroup' 'pod'", + # "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' CHOWN", "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' DAC_OVERRIDE", @@ -226,6 +232,11 @@ build { # Create 'crio.network' table (required for 'network_dir' settings). "sudo dasel put document --parser toml --file /etc/crio/crio.conf '.crio.network'", + # needed for networking + # this is done by the kubelet-config-helper.sh + # see https://github.com/nestybox/sysbox-pkgr/blob/b560194d516b300e9e201274a29348d3626055c1/k8s/scripts/kubelet-config-helper.sh#L833 + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.network.plugin_dirs.[]' '/opt/cni/bin'", + # "echo 'containers:231072:1048576' | sudo tee --append /etc/subuid", "echo 'containers:231072:1048576' | sudo tee --append /etc/subgid", diff --git a/variables.pkr.hcl b/variables.pkr.hcl index dc4b4cb..1116c27 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-02" + default = "pluraldev-03" } variable "ubuntu_version" { From 694ace95153ba018cc6dd8b92b0e7e1e53373913 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Wed, 9 Aug 2023 11:59:06 +0200 Subject: [PATCH 09/26] use prebuilt cri-o binaries Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 94 +++++++++++++++++++++++--------------------- sysbox-eks.pkr.hcl | 6 +-- variables.pkr.hcl | 7 +++- 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index ab44364..0479549 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -31,21 +31,21 @@ build { # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-sysbox "echo '>>> Sysbox'", "echo Downloading the Sysbox package", - "wget https://downloads.nestybox.com/sysbox/releases/v${var.sysbox_version}/sysbox-ce_${var.sysbox_version}-0.linux_amd64.deb", + "wget https://downloads.nestybox.com/sysbox/releases/v${var.sysbox_version}/sysbox-ce_${var.sysbox_version}-0.linux_${var.architecture}.deb", "echo Installing Sysbox package dependencies", "sudo apt-get install rsync -y", "echo Installing the Sysbox package", - "sudo dpkg --install ./sysbox-ce_*.linux_amd64.deb || true", # will fail due to missing dependencies, fixed in the next step + "sudo dpkg --install ./sysbox-ce_*.linux_${var.architecture}.deb || true", # will fail due to missing dependencies, fixed in the next step "echo 'Fixing the Sysbox package (installing dependencies)'", "sudo --preserve-env=DEBIAN_FRONTEND apt-get install --fix-broken --yes --no-install-recommends", "echo Cleaning up", - "rm ./sysbox-ce_*.linux_amd64.deb", + "rm ./sysbox-ce_*.linux_${var.architecture}.deb", ] } @@ -117,60 +117,66 @@ build { ## Uncomment this section to install from a patched CRI-O binary - # provisioner "file" { - # source = "crio" - # destination = "/home/ubuntu/crio" - # max_retries = 3 - # } - - # provisioner "shell" { - # inline = [ - # "echo >>> Installing prebuilt patched CRI-O", - # "sudo mv crio /usr/bin/crio", - # - # "echo Setting permissions", - # "sudo chmod u+x /usr/bin/crio" - # ] - # } + provisioner "file" { + source = "crio/${var.architecture}/v${var.k8s_version}/crio-patched" + destination = "/home/ubuntu/crio" + max_retries = 3 + } - ## Comment this section to install from a patched CRI-O binary provisioner "shell" { inline_shebang = "/usr/bin/env bash" - inline = [ "set -o pipefail -o errexit", - "echo '>>> Sysbox CRI-O patch'", - "echo Adding the Go backports repository", - "sudo apt-get install --yes --no-install-recommends software-properties-common", - "sudo add-apt-repository --yes ppa:longsleep/golang-backports", - - "echo Installing Go", - "sudo apt-get update", - # todo(maximsmol): lock the golang version - "sudo apt-get install --yes --no-install-recommends golang-go libgpgme-dev", - - "echo Cloning the patched CRI-O repository", - "git clone --branch v${var.k8s_version}-sysbox --depth 1 --shallow-submodules https://github.com/nestybox/cri-o.git cri-o", + "echo '>>> Installing prebuilt patched CRI-O'", + "sudo mv crio /usr/bin/crio", - "echo Building", - "cd cri-o", - "make binaries", - - "echo Installing the patched binary", - "sudo mv bin/crio /usr/bin/crio", + "echo Setting permissions", "sudo chmod u+x /usr/bin/crio", - - "echo Cleaning up", - "cd ..", - "rm -rf cri-o", - "echo Restarting CRI-O", "sudo systemctl restart crio" ] } + ## Comment this section to install from a patched CRI-O binary + # provisioner "shell" { + # inline_shebang = "/usr/bin/env bash" + + # inline = [ + # "set -o pipefail -o errexit", + + # "echo '>>> Sysbox CRI-O patch'", + # "echo Adding the Go backports repository", + # "sudo apt-get install --yes --no-install-recommends software-properties-common", + # "sudo add-apt-repository --yes ppa:longsleep/golang-backports", + + # "echo Installing Go", + # "sudo apt-get update", + # # todo(maximsmol): lock the golang version + # "sudo apt-get install --yes --no-install-recommends golang-go libgpgme-dev", + + # "echo Cloning the patched CRI-O repository", + # "git clone --branch v${var.k8s_version}-sysbox --depth 1 --shallow-submodules https://github.com/nestybox/cri-o.git cri-o", + + # "echo Building", + # "cd cri-o", + # "make binaries", + + # "echo Installing the patched binary", + # "sudo mv bin/crio /usr/bin/crio", + # "sudo chmod u+x /usr/bin/crio", + + + # "echo Cleaning up", + # "cd ..", + # "rm -rf cri-o", + + # "echo Restarting CRI-O", + # "sudo systemctl restart crio" + # ] + # } + provisioner "file" { source = "bootstrap.sh.patch" destination = "/home/ubuntu/bootstrap.sh.patch" @@ -192,7 +198,7 @@ build { "echo '>>> Doing basic CRI-O configuration'", "echo Installing Dasel", - "sudo curl --location https://github.com/TomWright/dasel/releases/download/v1.24.3/dasel_linux_amd64 --output /usr/local/bin/dasel", + "sudo curl --location https://github.com/TomWright/dasel/releases/download/v1.24.3/dasel_linux_${var.architecture} --output /usr/local/bin/dasel", "sudo chmod u+x /usr/local/bin/dasel", # Disable selinux for now. diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index b4bda2b..97f72cd 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -1,6 +1,6 @@ source "amazon-ebs" "ubuntu-eks" { ami_name = "${var.img_name}/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server" - ami_description = "Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}, amd64 image" + ami_description = "Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}" region = "us-west-2" instance_type = "t2.micro" @@ -10,7 +10,7 @@ source "amazon-ebs" "ubuntu-eks" { Linux = "Ubuntu" UbuntuRelease = split("-", var.ubuntu_version)[0] UbuntuVersion = split("-", var.ubuntu_version)[1] - Arch = "amd64" + Arch = "${var.architecture}" K8sVersion = var.k8s_version SysboxVersion = var.sysbox_version @@ -23,7 +23,7 @@ source "amazon-ebs" "ubuntu-eks" { source_ami_filter { filters = { - name = "ubuntu-eks/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server-*" + name = "ubuntu-eks/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-${var.architecture}-server-*" } most_recent = true owners = ["099720109477"] diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 1116c27..a35331f 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,12 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-03" + default = "pluraldev-04-prebuilt-crio" +} + +variable "architecture" { + type = string + default = "amd64" } variable "ubuntu_version" { From 4744ab6800c2cf1b1a6c6d712342b2fc9bd19c36 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Wed, 9 Aug 2023 14:52:57 +0200 Subject: [PATCH 10/26] cleanup build to upstream part1 Signed-off-by: David van der Spek --- .gitignore | 2 +- Makefile | 16 ++-- build-ubuntu.pkr.hcl | 199 ++++++++++++++++++++++++++++++------------- variables.pkr.hcl | 2 +- 4 files changed, 148 insertions(+), 71 deletions(-) diff --git a/.gitignore b/.gitignore index 198cbf2..a6a1fd3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ .DS_Store /scratch -/crio +/tmp diff --git a/Makefile b/Makefile index 3fc9b34..0286ca7 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ SYSBOX_VERSION=v0.6.2 -get-crio: - rm -rf ./crio - docker run --rm -it --platform linux/amd64 -v ./crio/amd64:/host/crio registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp -r /opt/crio-deploy/bin/* /host/crio/" - docker run --rm -it --platform linux/arm64 -v ./crio/arm64:/host/crio registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp -r /opt/crio-deploy/bin/* /host/crio/" -# remove tar.gz files - find ./crio/ -path '**/*.tar.gz' -delete -# remove empty directories - find ./crio/ -empty -type d -delete +get-files: + rm -rf ./tmp + mkdir -p ./tmp/sysbox/amd64/bin + mkdir -p ./tmp/sysbox/arm64/bin + mkdir -p ./tmp/crio/amd64 + mkdir -p ./tmp/crio/arm64 + docker run --rm -it --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/" + docker run --rm -it --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index 0479549..7e06401 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -12,76 +12,20 @@ build { # direction = "download" # } + # equivalent to install_package_deps() function + # TODO: seems like installing fuse removes fuse3. Which is needed by sysbox? According to arch package docs it hase fuse2 as a dependency. provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ "set -o pipefail -o errexit", "echo Updating apt", + "sudo apt-get -y install ca-certificates", "sudo apt-get update -y", + "sudo apt-get install -y rsync fuse iptables" ] } - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - "export DEBIAN_FRONTEND=noninteractive", - - # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-sysbox - "echo '>>> Sysbox'", - "echo Downloading the Sysbox package", - "wget https://downloads.nestybox.com/sysbox/releases/v${var.sysbox_version}/sysbox-ce_${var.sysbox_version}-0.linux_${var.architecture}.deb", - - "echo Installing Sysbox package dependencies", - - "sudo apt-get install rsync -y", - - "echo Installing the Sysbox package", - "sudo dpkg --install ./sysbox-ce_*.linux_${var.architecture}.deb || true", # will fail due to missing dependencies, fixed in the next step - - "echo 'Fixing the Sysbox package (installing dependencies)'", - - "sudo --preserve-env=DEBIAN_FRONTEND apt-get install --fix-broken --yes --no-install-recommends", - - "echo Cleaning up", - "rm ./sysbox-ce_*.linux_${var.architecture}.deb", - ] - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-shiftfs - "echo '>>> Shiftfs'", - - "echo Installing dependencies", - "sudo apt-get update", - "sudo apt-get install --yes --no-install-recommends make dkms git", - - "echo Detecting kernel version to determine the correct branch", - "export kernel_version=\"$(uname -r | sed --regexp-extended 's/([0-9]+\\.[0-9]+).*/\\1/g')\"", - "echo \"$kernel_version\"", - "declare -A kernel_to_branch=( [5.17]=k5.17 [5.16]=k5.16 [5.15]=k5.16 [5.14]=k5.13 [5.13]=k5.13 [5.10]=k5.10 [5.8]=k5.10 [5.4]=k5.4 )", - "export branch=\"$(echo $${kernel_to_branch[$kernel_version]})\"", - - "echo Cloning the repository branch: $branch", - "git clone --branch $branch --depth 1 --shallow-submodules https://github.com/toby63/shiftfs-dkms.git shiftfs", - "cd shiftfs", - - "echo Running the update script", - "./update1", - - "echo Building and installing", - "sudo make --file Makefile.dkms", - - "echo Cleaning up", - "cd ..", - "rm -rf shiftfs" - ] - } provisioner "shell" { inline_shebang = "/usr/bin/env bash" @@ -118,7 +62,7 @@ build { ## Uncomment this section to install from a patched CRI-O binary provisioner "file" { - source = "crio/${var.architecture}/v${var.k8s_version}/crio-patched" + source = "tmp/crio/${var.architecture}/v${var.k8s_version}/crio-patched" destination = "/home/ubuntu/crio" max_retries = 3 } @@ -177,6 +121,139 @@ build { # ] # } + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-shiftfs + "echo '>>> Shiftfs'", + + "echo Installing dependencies", + "sudo apt-get update", + "sudo apt-get install --yes --no-install-recommends make dkms git", + + "echo Detecting kernel version to determine the correct branch", + "export kernel_version=\"$(uname -r | sed --regexp-extended 's/([0-9]+\\.[0-9]+).*/\\1/g')\"", + "echo \"$kernel_version\"", + "declare -A kernel_to_branch=( [5.17]=k5.17 [5.16]=k5.16 [5.15]=k5.16 [5.14]=k5.13 [5.13]=k5.13 [5.10]=k5.10 [5.8]=k5.10 [5.4]=k5.4 )", + "export branch=\"$(echo $${kernel_to_branch[$kernel_version]})\"", + + "echo Cloning the repository branch: $branch", + "git clone --branch $branch --depth 1 --shallow-submodules https://github.com/toby63/shiftfs-dkms.git shiftfs", + "cd shiftfs", + + "echo Running the update script", + "./update1", + + "echo Building and installing", + "sudo make --file Makefile.dkms", + + "echo Cleaning up", + "cd ..", + "rm -rf shiftfs", + "sudo apt-get remove --yes --purge make dkms git" + ] + } + + # equivalent to copy_sysbox_to_host() function + provisioner "file" { + source = "tmp/sysbox/${var.architecture}/bin" + destination = "/home/ubuntu/" + } + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Moving Sysbox binaries to /usr/bin'", + "sudo mv /home/ubuntu/bin/* /usr/bin/", + ] + } + + # equivalent to copy_conf_to_host() function + provisioner "file" { + sources = ["tmp/sysbox/systemd/99-sysbox-sysctl.conf", "tmp/sysbox/systemd/50-sysbox-mod.conf"] + destination = "/home/ubuntu/" + } + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Moving Sysbox sysctl configs to /lib/sysctl.d/'", + "sudo mv /home/ubuntu/99-sysbox-sysctl.conf /lib/sysctl.d/99-sysbox-sysctl.conf", + "sudo mv /home/ubuntu/50-sysbox-mod.conf /lib/sysctl.d/50-sysbox-mod.conf", + ] + } + + # equivalent to copy_systemd_units_to_host() function + provisioner "file" { + sources = ["tmp/sysbox/systemd/sysbox.service", "tmp/sysbox/systemd/sysbox-mgr.service", "tmp/sysbox/systemd/sysbox-fs.service"] + destination = "/home/ubuntu/" + } + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", + + "echo '>>> Moving Sysbox systemd units to /lib/systemd/system/'", + "sudo mv /home/ubuntu/sysbox.service /lib/systemd/system/sysbox.service", + "sudo mv /home/ubuntu/sysbox-mgr.service /lib/systemd/system/sysbox-mgr.service", + "sudo mv /home/ubuntu/sysbox-fs.service /lib/systemd/system/sysbox-fs.service", + + "echo '>>> Enabling Sysbox systemd units'", + "sudo systemctl daemon-reload", + "sudo systemctl enable sysbox.service", + "sudo systemctl enable sysbox-mgr.service", + "sudo systemctl enable sysbox-fs.service", + ] + } + + # equivalent to apply_conf() + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "sudo echo 'Configuring host sysctls ...'", + "sudo sysctl -p '/lib/sysctl.d/99-sysbox-sysctl.conf'", + ] + } + + # equivalent to start_sysbox() + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "sudo echo 'Starting CE ...'", + "sudo systemctl restart sysbox", + "sudo systemctl is-active --quiet sysbox", + ] + } + + + + # provisioner "shell" { + # inline_shebang = "/usr/bin/env bash" + # inline = [ + # "set -o pipefail -o errexit", + # "export DEBIAN_FRONTEND=noninteractive", + + # # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-sysbox + # "echo '>>> Sysbox'", + # "echo Downloading the Sysbox package", + # "wget https://downloads.nestybox.com/sysbox/releases/v${var.sysbox_version}/sysbox-ce_${var.sysbox_version}-0.linux_${var.architecture}.deb", + + # "echo Installing the Sysbox package", + # "sudo dpkg --install ./sysbox-ce_*.linux_${var.architecture}.deb || true", # will fail due to missing dependencies, fixed in the next step + + # "echo 'Fixing the Sysbox package (installing dependencies)'", + + # "sudo --preserve-env=DEBIAN_FRONTEND apt-get install --fix-broken --yes --no-install-recommends", + + # "echo Cleaning up", + # "rm ./sysbox-ce_*.linux_${var.architecture}.deb", + # ] + # } + provisioner "file" { source = "bootstrap.sh.patch" destination = "/home/ubuntu/bootstrap.sh.patch" diff --git a/variables.pkr.hcl b/variables.pkr.hcl index a35331f..eaaa060 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-04-prebuilt-crio" + default = "pluraldev-05-cleanup-part1" } variable "architecture" { From 366c5cc62563c87074635026af9b1d76cdd1e5a8 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Wed, 9 Aug 2023 16:16:35 +0200 Subject: [PATCH 11/26] build script cleanup part 2 Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 208 ++++++++++++++-------------------- scripts/config_subid_range.sh | 131 +++++++++++++++++++++ variables.pkr.hcl | 2 +- 3 files changed, 219 insertions(+), 122 deletions(-) create mode 100755 scripts/config_subid_range.sh diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index 7e06401..bd35016 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -5,12 +5,12 @@ build { ] - # Can be used to gen the current bootstrap.sh to update the patch -# provisioner "file" { -# source = "/usr/local/share/eks/bootstrap.sh" -# destination = "current_bootstrap.sh" -# direction = "download" -# } + # # Can be used to gen the current bootstrap.sh to update the patch + # provisioner "file" { + # source = "/usr/local/share/eks/bootstrap.sh" + # destination = "current_bootstrap.sh" + # direction = "download" + # } # equivalent to install_package_deps() function # TODO: seems like installing fuse removes fuse3. Which is needed by sysbox? According to arch package docs it hase fuse2 as a dependency. @@ -26,6 +26,9 @@ build { ] } + ################### + ## Install CRI-O ## + ################### provisioner "shell" { inline_shebang = "/usr/bin/env bash" @@ -59,14 +62,13 @@ build { ] } - ## Uncomment this section to install from a patched CRI-O binary + # part of do_install_crio() function provisioner "file" { source = "tmp/crio/${var.architecture}/v${var.k8s_version}/crio-patched" destination = "/home/ubuntu/crio" max_retries = 3 } - provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ @@ -83,44 +85,68 @@ build { ] } - ## Comment this section to install from a patched CRI-O binary - # provisioner "shell" { - # inline_shebang = "/usr/bin/env bash" + # equivalent to config_crio() function + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + "set -o pipefail -o errexit", - # inline = [ - # "set -o pipefail -o errexit", + # Much of the rest of this is from inside the Sysbox K8s installer image + "echo '>>> Doing basic CRI-O configuration'", - # "echo '>>> Sysbox CRI-O patch'", - # "echo Adding the Go backports repository", - # "sudo apt-get install --yes --no-install-recommends software-properties-common", - # "sudo add-apt-repository --yes ppa:longsleep/golang-backports", + "echo Installing Dasel", + "sudo curl --location https://github.com/TomWright/dasel/releases/download/v1.24.3/dasel_linux_${var.architecture} --output /usr/local/bin/dasel", + "sudo chmod u+x /usr/local/bin/dasel", - # "echo Installing Go", - # "sudo apt-get update", - # # todo(maximsmol): lock the golang version - # "sudo apt-get install --yes --no-install-recommends golang-go libgpgme-dev", + # Disable selinux for now. + "sudo dasel put bool --parser toml --file /etc/crio/crio.conf 'crio.runtime.selinux' false", - # "echo Cloning the patched CRI-O repository", - # "git clone --branch v${var.k8s_version}-sysbox --depth 1 --shallow-submodules https://github.com/nestybox/cri-o.git cri-o", + # # Add user "containers" to the /etc/subuid and /etc/subgid files + # NOTE: this is done in the next step with config_subid_range.sh - # "echo Building", - # "cd cri-o", - # "make binaries", + # Set capabilities to match default caps in containerd/docker + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' CHOWN", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' DAC_OVERRIDE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' FSETID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' FOWNER", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETUID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETGID", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETPCAP", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETFCAP", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' NET_BIND_SERVICE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' KILL", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' AUDIT_WRITE", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' NET_RAW", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SYS_CHROOT", + "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' MKNOD", - # "echo Installing the patched binary", - # "sudo mv bin/crio /usr/bin/crio", - # "sudo chmod u+x /usr/bin/crio", + # Create 'crio.image' table (required for 'pause_image' settings). + "sudo dasel put document --parser toml --file /etc/crio/crio.conf '.crio.image'", + # Create 'crio.network' table (required for 'network_dir' settings). + "sudo dasel put document --parser toml --file /etc/crio/crio.conf '.crio.network'", - # "echo Cleaning up", - # "cd ..", - # "rm -rf cri-o", + # CRI-O puts a default limit of 1024 processes per pod; this is too small for + # Sysbox pods, since these run sometimes complex software such as Docker, + # K8s, etc. Thus we increase this to 16K processes per pod. Since the max + # limit for Linux is 4M (see /proc/sys/kernel/pid_max), this allows up to + # ~256 Sysbox containers each consuming 16K processes on a given host. It + # also constraints a malicious container executing a fork bomb to 16K + # processes, well below the kernel's max pid limit. + "sudo dasel put int --parser toml --file /etc/crio/crio.conf 'crio.runtime.pids_limit' 16384", + ] + } - # "echo Restarting CRI-O", - # "sudo systemctl restart crio" - # ] - # } + # equivalent to get_subid_limits() and config_subid_range() functions + provisioner "shell" { + script = "scripts/config_subid_range.sh" + } + #################### + ## Install Sysbox ## + #################### + + # equivalent to install_shiftfs() function provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ @@ -229,125 +255,65 @@ build { ] } - - - # provisioner "shell" { - # inline_shebang = "/usr/bin/env bash" - # inline = [ - # "set -o pipefail -o errexit", - # "export DEBIAN_FRONTEND=noninteractive", - - # # https://github.com/nestybox/sysbox/blob/b25fe4a3f9a6501992f8bb3e28d206302de9f33b/docs/user-guide/install-package.md#installing-sysbox - # "echo '>>> Sysbox'", - # "echo Downloading the Sysbox package", - # "wget https://downloads.nestybox.com/sysbox/releases/v${var.sysbox_version}/sysbox-ce_${var.sysbox_version}-0.linux_${var.architecture}.deb", - - # "echo Installing the Sysbox package", - # "sudo dpkg --install ./sysbox-ce_*.linux_${var.architecture}.deb || true", # will fail due to missing dependencies, fixed in the next step - - # "echo 'Fixing the Sysbox package (installing dependencies)'", - - # "sudo --preserve-env=DEBIAN_FRONTEND apt-get install --fix-broken --yes --no-install-recommends", - - # "echo Cleaning up", - # "rm ./sysbox-ce_*.linux_${var.architecture}.deb", - # ] - # } - - provisioner "file" { - source = "bootstrap.sh.patch" - destination = "/home/ubuntu/bootstrap.sh.patch" - } - - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "sudo mv /home/ubuntu/bootstrap.sh.patch /usr/local/share/eks/bootstrap.sh.patch", - ] - } - + # equivalent to config_crio_for_sysbox() function provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ "set -o pipefail -o errexit", - # Much of the rest of this is from inside the Sysbox K8s installer image - "echo '>>> Doing basic CRI-O configuration'", - - "echo Installing Dasel", - "sudo curl --location https://github.com/TomWright/dasel/releases/download/v1.24.3/dasel_linux_${var.architecture} --output /usr/local/bin/dasel", - "sudo chmod u+x /usr/local/bin/dasel", - - # Disable selinux for now. - "sudo dasel put bool --parser toml --file /etc/crio/crio.conf 'crio.runtime.selinux' false", + "echo 'Adding Sysbox to CRI-O config ...'", # overlayfs with metacopy=on improves startup time of CRI-O rootless containers significantly "sudo dasel put string --parser toml --file /etc/crio/crio.conf 'crio.storage_driver' 'overlay'", "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.storage_option.[]' 'overlay.mountopt=metacopy=on'", + # Add Sysbox to CRI-O's runtime list + "sudo dasel put object --parser toml -m 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", + "sudo dasel put string --parser toml -m 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", + ] + } + + # equivalent to adjust_crio_config_dependencies() function (from kubelet-config-helpe.sh that usually runs at runtime) + # see https://github.com/nestybox/sysbox-pkgr/blob/b560194d516b300e9e201274a29348d3626055c1/k8s/scripts/kubelet-config-helper.sh#L861 + # see https://github.com/nestybox/sysbox-pkgr/blob/b560194d516b300e9e201274a29348d3626055c1/k8s/scripts/kubelet-config-helper.sh#L833 + provisioner "shell" { + inline_shebang = "/usr/bin/env bash" + inline = [ + # todo(maximsmol): do this only when K8s is configured without systemd cgroups (from sysbox todos) # this is done by the kubelet-config-helper.sh # see https://github.com/nestybox/sysbox-pkgr/blob/b560194d516b300e9e201274a29348d3626055c1/k8s/scripts/kubelet-config-helper.sh#L861 "sudo dasel put string --parser toml --file /etc/crio/crio.conf 'crio.runtime.cgroup_manager' 'cgroupfs'", "sudo dasel put string --parser toml --file /etc/crio/crio.conf 'crio.runtime.conmon_cgroup' 'pod'", - # - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' CHOWN", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' DAC_OVERRIDE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' FSETID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' FOWNER", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETUID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETGID", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETPCAP", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SETFCAP", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' NET_BIND_SERVICE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' KILL", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' AUDIT_WRITE", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' NET_RAW", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' SYS_CHROOT", - "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.runtime.default_capabilities.[]' MKNOD", - # - "sudo dasel put int --parser toml --file /etc/crio/crio.conf 'crio.runtime.pids_limit' 16384", - - # Create 'crio.image' table (required for 'pause_image' settings). - "sudo dasel put document --parser toml --file /etc/crio/crio.conf '.crio.image'", - - # Create 'crio.network' table (required for 'network_dir' settings). - "sudo dasel put document --parser toml --file /etc/crio/crio.conf '.crio.network'", - # needed for networking # this is done by the kubelet-config-helper.sh # see https://github.com/nestybox/sysbox-pkgr/blob/b560194d516b300e9e201274a29348d3626055c1/k8s/scripts/kubelet-config-helper.sh#L833 "sudo dasel put string --parser toml --file /etc/crio/crio.conf -m 'crio.network.plugin_dirs.[]' '/opt/cni/bin'", - - # - "echo 'containers:231072:1048576' | sudo tee --append /etc/subuid", - "echo 'containers:231072:1048576' | sudo tee --append /etc/subgid", - # /usr/local/share/eks/bootstrap.sh is symlinked to /etc/eks/boostrap.sh - "sudo patch --backup /usr/local/share/eks/bootstrap.sh /usr/local/share/eks/bootstrap.sh.patch" ] } + # TODO: this is sus as it isn't done upstream provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ "set -o pipefail -o errexit", - "echo '>>> Configuring CRI-O for Sysbox'", - - "echo Adding Sysbox to CRI-O runtimes", - "sudo dasel put object --parser toml -m 'crio.runtime.runtimes.sysbox-runc' --file /etc/crio/crio.conf --type string 'runtime_path=/usr/bin/sysbox-runc' --type string 'runtime_type=oci'", - "sudo dasel put string --parser toml -m 'crio.runtime.runtimes.sysbox-runc.allowed_annotations.[0]' --file /etc/crio/crio.conf 'io.kubernetes.cri-o.userns-mode'", + "echo '>>> Removing /etc/cni/net.d'", + "sudo rm -r /etc/cni/net.d/", ] } + provisioner "file" { + source = "bootstrap.sh.patch" + destination = "/home/ubuntu/bootstrap.sh.patch" + } + provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ - "set -o pipefail -o errexit", - - "echo '>>> Removing /etc/cni/net.d'", - "sudo rm -r /etc/cni/net.d/", + "sudo mv /home/ubuntu/bootstrap.sh.patch /usr/local/share/eks/bootstrap.sh.patch", + "sudo patch --backup /usr/local/share/eks/bootstrap.sh /usr/local/share/eks/bootstrap.sh.patch" ] } } diff --git a/scripts/config_subid_range.sh b/scripts/config_subid_range.sh new file mode 100755 index 0000000..2a85cfc --- /dev/null +++ b/scripts/config_subid_range.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + + +# +# Subid default values. +# +# Sysbox supports up 4K sys contaienrs per K8s node, each with 64K subids. +# +# Historical note: prior to Docker's acquisition of Nesytbox, Sysbox-CE was +# limited to 16-pods-per-node via variable subid_alloc_min_range below, whereas +# Sysbox-EE was limited to 4K-pods-per-node. After Docker's acquisition of +# Nestybox (05/22) Sysbox-EE is no longer being offered and therefore Docker has +# decided to lift the Sysbox-CE limit to encourage adoption of Sysbox on K8s +# clusters (the limit will now be 4K-pods-per-node as it was in Sysbox-EE). +# +subid_alloc_min_start=100000 +subid_alloc_min_range=268435456 +subid_alloc_max_end=4294967295 + +# We use CRI-O's default user "containers" for the sub-id range (rather than +# user "sysbox"). +subid_user="containers" +subid_def_file="/etc/login.defs" +subuid_file="/etc/subuid" +subgid_file="/etc/subgid" + +function get_subid_limits() { + + # Get subid defaults from /etc/login.defs + + subuid_min=$subid_alloc_min_start + subuid_max=$subid_alloc_max_end + subgid_min=$subid_alloc_min_start + subgid_max=$subid_alloc_max_end + + if [ ! -f $subid_def_file ]; then + return + fi + + set +e + res=$(grep "^SUB_UID_MIN" $subid_def_file >/dev/null 2>&1) + if [ $? -eq 0 ]; then + subuid_min=$(echo $res | cut -d " " -f2) + fi + + res=$(grep "^SUB_UID_MAX" $subid_def_file >/dev/null 2>&1) + if [ $? -eq 0 ]; then + subuid_max=$(echo $res | cut -d " " -f2) + fi + + res=$(grep "^SUB_GID_MIN" $subid_def_file >/dev/null 2>&1) + if [ $? -eq 0 ]; then + subgid_min=$(echo $res | cut -d " " -f2) + fi + + res=$(grep "^SUB_GID_MAX" $subid_def_file >/dev/null 2>&1) + if [ $? -eq 0 ]; then + subgid_max=$(echo $res | cut -d " " -f2) + fi + set -e +} + +function config_subid_range() { + local subid_file=$1 + local subid_size=$2 + local subid_min=$3 + local subid_max=$4 + + if [ ! -f $subid_file ] || [ ! -s $subid_file ]; then + echo "$subid_user:$subid_min:$subid_size" >"${subid_file}" + return + fi + + readarray -t subid_entries <"${subid_file}" + + # if a large enough subid config already exists for user $subid_user, there + # is nothing to do. + + for entry in "${subid_entries[@]}"; do + user=$(echo $entry | cut -d ":" -f1) + start=$(echo $entry | cut -d ":" -f2) + size=$(echo $entry | cut -d ":" -f3) + + if [[ "$user" == "$subid_user" ]] && [ "$size" -ge "$subid_size" ]; then + return + fi + done + + # Sort subid entries by start range + declare -a sorted_subids + if [ ${#subid_entries[@]} -gt 0 ]; then + readarray -t sorted_subids < <(echo "${subid_entries[@]}" | tr " " "\n" | tr ":" " " | sort -n -k 2) + fi + + # allocate a range of subid_alloc_range size + hole_start=$subid_min + + for entry in "${sorted_subids[@]}"; do + start=$(echo $entry | cut -d " " -f2) + size=$(echo $entry | cut -d " " -f3) + + hole_end=$start + + if [ $hole_end -ge $hole_start ] && [ $((hole_end - hole_start)) -ge $subid_size ]; then + echo "$subid_user:$hole_start:$subid_size" >>$subid_file + return + fi + + hole_start=$((start + size)) + done + + hole_end=$subid_max + if [ $((hole_end - hole_start)) -lt $subid_size ]; then + echo "failed to allocate $subid_size sub ids in range $subid_min:$subid_max" + return + else + echo "$subid_user:$hole_start:$subid_size" >>$subid_file + return + fi +} + +function main() { + sudo su - + get_subid_limits + config_subid_range "$subuid_file" "$subid_alloc_min_range" "$subuid_min" "$subuid_max" + config_subid_range "$subgid_file" "$subid_alloc_min_range" "$subgid_min" "$subgid_max" +} diff --git a/variables.pkr.hcl b/variables.pkr.hcl index eaaa060..1501cff 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-05-cleanup-part1" + default = "pluraldev-05-cleanup-part2" } variable "architecture" { From 93867fac91da439ab2ba2924579fcce37707c8ad Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Wed, 9 Aug 2023 17:13:41 +0200 Subject: [PATCH 12/26] fix configuring subuid and subgid using script Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 3 ++- scripts/config_subid_range.sh | 3 ++- variables.pkr.hcl | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index bd35016..89fd560 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -139,7 +139,8 @@ build { # equivalent to get_subid_limits() and config_subid_range() functions provisioner "shell" { - script = "scripts/config_subid_range.sh" + script = "scripts/config_subid_range.sh" + execute_command = "chmod +x {{ .Path }}; sudo sh -c '{{ .Vars }} {{ .Path }}'" } #################### diff --git a/scripts/config_subid_range.sh b/scripts/config_subid_range.sh index 2a85cfc..814944b 100755 --- a/scripts/config_subid_range.sh +++ b/scripts/config_subid_range.sh @@ -124,8 +124,9 @@ function config_subid_range() { } function main() { - sudo su - get_subid_limits config_subid_range "$subuid_file" "$subid_alloc_min_range" "$subuid_min" "$subuid_max" config_subid_range "$subgid_file" "$subid_alloc_min_range" "$subgid_min" "$subgid_max" } + +main diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 1501cff..7ce9bf4 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-05-cleanup-part2" + default = "pluraldev-05-cleanup-part3" } variable "architecture" { From 91deb44639264f277da12b3ada0fa8b524b5dc24 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Wed, 9 Aug 2023 17:59:47 +0200 Subject: [PATCH 13/26] cleanup crio install to use upstream binary method Signed-off-by: David van der Spek --- Makefile | 2 +- build-ubuntu.pkr.hcl | 101 ++++++++++++++-------------- scripts/config_containers_common.sh | 66 ++++++++++++++++++ sysbox-eks.pkr.hcl | 2 +- variables.pkr.hcl | 2 +- 5 files changed, 119 insertions(+), 54 deletions(-) create mode 100755 scripts/config_containers_common.sh diff --git a/Makefile b/Makefile index 0286ca7..0d35fe9 100644 --- a/Makefile +++ b/Makefile @@ -6,5 +6,5 @@ get-files: mkdir -p ./tmp/sysbox/arm64/bin mkdir -p ./tmp/crio/amd64 mkdir -p ./tmp/crio/arm64 - docker run --rm -it --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/" + docker run --rm -it --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/ && cp -r /opt/crio-deploy/scripts/ /host/crio/scripts/" docker run --rm -it --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index 89fd560..ff4940a 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -30,58 +30,68 @@ build { ## Install CRI-O ## ################### + + # equivalent to deploy_crio_installer_service() function + provisioner "file" { + sources = [ + "tmp/crio/${var.architecture}/v${var.k8s_version}/crio-patched", + "tmp/crio/${var.architecture}/v${var.k8s_version}/cri-o.${var.architecture}.tar.gz", + "tmp/crio/scripts/crio-extractor.sh", + "tmp/crio/config/etc_cni_net.d_200-loopback.conf", + "tmp/crio/config/etc_containers_registries.conf.d_000-shortnames.conf", + "tmp/crio/config/etc_containers_storage.conf", + "tmp/crio/config/etc_containers_registries.conf", + "tmp/crio/config/etc_containers_registries.d_default.yaml", + "tmp/crio/config/etc_containers_policy.json", + ] + destination = "/home/ubuntu/" + max_retries = 3 + } provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ - "set -o pipefail -o errexit", - - # https://github.com/cri-o/cri-o/blob/a68a72071e5004be78fe2b1b98cb3bfa0e51b74b/install.md#apt-based-operating-systems - "echo '>>> CRI-O'", - - # fixme(maximsmol): take into account ${ubuntu_version} - "export OS='xUbuntu_20.04'", - "export VERSION='${var.k8s_version}'", - - "echo Adding repositories", - "echo \"deb [signed-by=/usr/share/keyrings/libcontainers-archive-keyring.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /\" | sudo dd status=none of=/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list", - "echo \"deb [signed-by=/usr/share/keyrings/libcontainers-crio-archive-keyring.gpg] http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /\" | sudo dd status=none of=/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list", - - "echo Adding keys", - "mkdir --parents /usr/share/keyrings", - "curl --location https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$VERSION/$OS/Release.key | sudo gpg --dearmor --output /usr/share/keyrings/libcontainers-archive-keyring.gpg", - "curl --location https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo gpg --dearmor --output /usr/share/keyrings/libcontainers-crio-archive-keyring.gpg", - - "echo Updating apt", - "sudo apt-get update", - - "echo Installing CRI-O", - "sudo apt-get install --yes --no-install-recommends cri-o cri-o-runc cri-tools", - - "echo Enabling CRI-O at startup", - "sudo systemctl enable crio" + "sudo mv cri-o.${var.architecture}.tar.gz /usr/local/bin/cri-o.${var.architecture}.tar.gz", + "sudo mv crio-patched /usr/local/bin/crio-patched", + "sudo chmod +x crio-extractor.sh && sudo mv crio-extractor.sh /usr/local/bin/crio-extractor.sh", + + "mkdir -p crio/config", + "mv etc_cni_net.d_200-loopback.conf crio/config/etc_cni_net.d_200-loopback.conf", + "mv etc_containers_registries.conf.d_000-shortnames.conf crio/config/etc_containers_registries.conf.d_000-shortnames.conf", + "mv etc_containers_storage.conf crio/config/etc_containers_storage.conf", + "mv etc_containers_registries.conf crio/config/etc_containers_registries.conf", + "mv etc_containers_registries.d_default.yaml crio/config/etc_containers_registries.d_default.yaml", + "mv etc_containers_policy.json crio/config/etc_containers_policy.json", ] } - ## Uncomment this section to install from a patched CRI-O binary - # part of do_install_crio() function - provisioner "file" { - source = "tmp/crio/${var.architecture}/v${var.k8s_version}/crio-patched" - destination = "/home/ubuntu/crio" - max_retries = 3 + # equivalent to config_containers_common() function + provisioner "shell" { + script = "scripts/config_containers_common.sh" + execute_command = "chmod +x {{ .Path }}; sudo sh -c '{{ .Vars }} {{ .Path }}'" } + + # equivalent to install_crio() function provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ - "set -o pipefail -o errexit", + # Extract and install the CRI-O (and related dependencies) binaries + "pushd '/usr/local/bin'", + "sudo tar -xvf 'cri-o.${var.architecture}.tar.gz'", + "sudo rm -r 'cri-o.${var.architecture}.tar.gz'", + "pushd cri-o", + + "sudo sh -c \"/usr/local/bin/crio-extractor.sh install '/usr/local/bin'\"", + "sudo rm -r /usr/local/bin/cri-o", - "echo '>>> Installing prebuilt patched CRI-O'", - "sudo mv crio /usr/bin/crio", + # Replace the stock CRI-O binary with the one that has the uid mapping patch + # required by Sysbox. + "sudo mv /usr/local/bin/crio-patched /usr/local/bin/crio", - "echo Setting permissions", - "sudo chmod u+x /usr/bin/crio", + # Remove the CRI-O extractor script since it is no longer needed. + "sudo rm /usr/local/bin/crio-extractor.sh", - "echo Restarting CRI-O", - "sudo systemctl restart crio" + "sudo systemctl enable crio", + "echo 'CRI-O installation done.'", ] } @@ -294,22 +304,11 @@ build { ] } - # TODO: this is sus as it isn't done upstream - provisioner "shell" { - inline_shebang = "/usr/bin/env bash" - inline = [ - "set -o pipefail -o errexit", - - "echo '>>> Removing /etc/cni/net.d'", - "sudo rm -r /etc/cni/net.d/", - ] - } - + # patch the bootstrap.sh to support cri-o and set it as the default provisioner "file" { source = "bootstrap.sh.patch" destination = "/home/ubuntu/bootstrap.sh.patch" } - provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ diff --git a/scripts/config_containers_common.sh b/scripts/config_containers_common.sh new file mode 100755 index 0000000..8e6e19d --- /dev/null +++ b/scripts/config_containers_common.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +# The instructions in this function are typically executed as part of the +# containers-common's deb-pkg installation (which is a dependency of the cri-o +# pkg) by creating the default config files required for cri-o operations. +# However, these config files are not part of the cri-o tar file that +# we're relying on in this installation process, so we must explicitly create +# this configuration state as part of the installation process. +function config_containers_common() { + + local config_files="/home/ubuntu/crio/config" + local containers_dir="/etc/containers" + mkdir -p "$containers_dir" + + # Create a default system-wide registries.conf file and associated drop-in + # dir if not already present. + local reg_file="${containers_dir}/registries.conf" + if [ ! -f "$reg_file" ]; then + mv "${config_files}/etc_containers_registries.conf" "${reg_file}" + fi + + local reg_dropin_dir="${containers_dir}/registries.conf.d" + mkdir -p "$reg_dropin_dir" + + # Copy registry shortname config + local shortnames_conf_file="${reg_dropin_dir}/000-shortnames.conf" + if [ ! -f "$shortnames_conf_file" ]; then + mv "${config_files}/etc_containers_registries.conf.d_000-shortnames.conf" "${shortnames_conf_file}" + fi + + # Create a default registry-configuration file if not already present. + local reg_dir="${containers_dir}/registries.d" + mkdir -p "$reg_dir" + + local reg_def_file="${reg_dir}/default.yaml" + if [ ! -f "$reg_def_file" ]; then + mv "${config_files}/etc_containers_registries.d_default.yaml" "${reg_def_file}" + fi + + # Create a default storage.conf file if not already present. + local storage_conf_file="${containers_dir}/storage.conf" + if [ ! -f "$storage_conf_file" ]; then + mv "${config_files}/etc_containers_storage.conf" "${storage_conf_file}" + fi + + # Create a default policy.json file if not already present. + local policy_file="${containers_dir}/policy.json" + if [ ! -f "$policy_file" ]; then + mv "${config_files}/etc_containers_policy.json" "${policy_file}" + fi + + # Copy the default loopback CNI config file + local cni_dir="/etc/cni/net.d" + mkdir -p "$cni_dir" + + local lb_file="${cni_dir}/200-loopback.conf" + if [ ! -f "$lb_file" ]; then + mv "${config_files}/etc_cni_net.d_200-loopback.conf" "${lb_file}" + fi +} + +config_containers_common diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index 97f72cd..b84aa16 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -2,7 +2,7 @@ source "amazon-ebs" "ubuntu-eks" { ami_name = "${var.img_name}/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server" ami_description = "Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}" - region = "us-west-2" + region = "us-east-2" instance_type = "t2.micro" ami_regions = var.aws_target_regions diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 7ce9bf4..21e1dcb 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-05-cleanup-part3" + default = "pluraldev-05-cleanup-part4-crio-binary" } variable "architecture" { From 05bb14a7973bdf61bdeecac4c730757d7ba92bec Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 10 Aug 2023 12:29:38 +0200 Subject: [PATCH 14/26] fix cri-o installation for tar file Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 4 +++- sysbox-eks.pkr.hcl | 2 +- variables.pkr.hcl | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index ff4940a..bd8db23 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -80,7 +80,7 @@ build { "sudo rm -r 'cri-o.${var.architecture}.tar.gz'", "pushd cri-o", - "sudo sh -c \"/usr/local/bin/crio-extractor.sh install '/usr/local/bin'\"", + "sudo sh -c \"/usr/local/bin/crio-extractor.sh install '/usr/local'\"", "sudo rm -r /usr/local/bin/cri-o", # Replace the stock CRI-O binary with the one that has the uid mapping patch @@ -91,6 +91,8 @@ build { "sudo rm /usr/local/bin/crio-extractor.sh", "sudo systemctl enable crio", + "sudo systemctl restart crio", + "sudo systemctl is-active --quiet crio", "echo 'CRI-O installation done.'", ] } diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index b84aa16..97f72cd 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -2,7 +2,7 @@ source "amazon-ebs" "ubuntu-eks" { ami_name = "${var.img_name}/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server" ami_description = "Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}" - region = "us-east-2" + region = "us-west-2" instance_type = "t2.micro" ami_regions = var.aws_target_regions diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 21e1dcb..e6f5610 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-05-cleanup-part4-crio-binary" + default = "pluraldev-05-cleanup-part4-crio-binary-fix2" } variable "architecture" { From e617047480b69719107db77e21d3689285ce6803 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 10 Aug 2023 12:36:43 +0200 Subject: [PATCH 15/26] fix location of 50-sysbox-mod.conf Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 2 +- variables.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index bd8db23..f410650 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -222,7 +222,7 @@ build { "echo '>>> Moving Sysbox sysctl configs to /lib/sysctl.d/'", "sudo mv /home/ubuntu/99-sysbox-sysctl.conf /lib/sysctl.d/99-sysbox-sysctl.conf", - "sudo mv /home/ubuntu/50-sysbox-mod.conf /lib/sysctl.d/50-sysbox-mod.conf", + "sudo mv /home/ubuntu/50-sysbox-mod.conf /usr/lib/modules-load.d/50-sysbox-mod.conf", ] } diff --git a/variables.pkr.hcl b/variables.pkr.hcl index e6f5610..eff06e7 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -5,7 +5,7 @@ variable "aws_target_regions" { variable "img_name" { type = string - default = "pluraldev-05-cleanup-part4-crio-binary-fix2" + default = "pluraldev-05-cleanup-part5" } variable "architecture" { From 5a2763cf0ae40b82c1390c7036faabe532b5ffa8 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Thu, 10 Aug 2023 12:53:56 +0200 Subject: [PATCH 16/26] cleanup todo comment Signed-off-by: David van der Spek --- build-ubuntu.pkr.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/build-ubuntu.pkr.hcl b/build-ubuntu.pkr.hcl index f410650..83c512c 100644 --- a/build-ubuntu.pkr.hcl +++ b/build-ubuntu.pkr.hcl @@ -13,7 +13,6 @@ build { # } # equivalent to install_package_deps() function - # TODO: seems like installing fuse removes fuse3. Which is needed by sysbox? According to arch package docs it hase fuse2 as a dependency. provisioner "shell" { inline_shebang = "/usr/bin/env bash" inline = [ From a5a12dea5c6aad5e9a55c4469acae7716de3d395 Mon Sep 17 00:00:00 2001 From: David van der Spek <28541758+DavidSpek@users.noreply.github.com> Date: Thu, 10 Aug 2023 16:53:46 +0200 Subject: [PATCH 17/26] ci: add packer build automation (#2) Signed-off-by: David van der Spek --- .github/workflows/build.yaml | 77 ++++++++++++++++++++++++++++++++++ .github/workflows/linting.yaml | 20 --------- Makefile | 13 +++++- plugins.pkr.hcl | 2 +- sysbox-eks.pkr.hcl | 14 ++++--- variables.pkr.hcl | 27 +++++++++++- 6 files changed, 123 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/build.yaml delete mode 100644 .github/workflows/linting.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..4a21a92 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,77 @@ +name: Packer Build +on: + push: + branches: + - main + pull_request: + branches: + - main +jobs: + packer_build_eks: + name: Build EKS AMI + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + strategy: + matrix: + k8s_version: ["1.23", "1.24", "1.25", "1.26"] + ubuntu_version: ["focal-20.04"] + architecture: ["amd64", "arm64"] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + if: github.event_name != 'pull_request' + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::654897662046:role/github-actions/plural-sysbox-amis-packer + role-session-name: SysboxAmisPacker + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Get files + run: make get-files + - name: Setup `packer` + uses: hashicorp/setup-packer@main + id: setup + with: + version: 1.9.2 + - name: Run `packer init` + id: init + run: "packer init ." + - name: Run `packer validate` + id: validate + run: "packer validate ." + - name: Run `packer build` + id: build + if: github.event_name != 'pull_request' + env: + PKR_VAR_k8s_version: ${{ matrix.k8s_version }} + PKR_VAR_ubuntu_version: ${{ matrix.ubuntu_version }} + PKR_VAR_architecture: ${{ matrix.architecture }} + run: "packer build ." + # trivy-scan: + # name: Trivy fs scan + # runs-on: ubuntu-latest + # permissions: + # contents: read # for actions/checkout to fetch code + # security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + # actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + # steps: + # - name: Checkout code + # uses: actions/checkout@v3 + # - name: Run Trivy vulnerability scanner in fs mode + # uses: aquasecurity/trivy-action@master + # with: + # scan-type: 'fs' + # hide-progress: false + # format: 'sarif' + # output: 'trivy-results.sarif' + # scanners: 'vuln,secret' + # ignore-unfixed: true + # #severity: 'CRITICAL,HIGH' + # - name: Upload Trivy scan results to GitHub Security tab + # uses: github/codeql-action/upload-sarif@v2 + # with: + # sarif_file: 'trivy-results.sarif' diff --git a/.github/workflows/linting.yaml b/.github/workflows/linting.yaml deleted file mode 100644 index 91013f8..0000000 --- a/.github/workflows/linting.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Linting -on: - - push -jobs: - pre-commit: - name: pre-commit - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Init Packer - uses: hashicorp/packer-github-actions@master - with: - command: init - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: pre-commit - uses: pre-commit/action@v2.0.3 diff --git a/Makefile b/Makefile index 0d35fe9..6a12504 100644 --- a/Makefile +++ b/Makefile @@ -6,5 +6,14 @@ get-files: mkdir -p ./tmp/sysbox/arm64/bin mkdir -p ./tmp/crio/amd64 mkdir -p ./tmp/crio/arm64 - docker run --rm -it --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/ && cp -r /opt/crio-deploy/scripts/ /host/crio/scripts/" - docker run --rm -it --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" + docker run --rm --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/ && cp -r /opt/crio-deploy/scripts/ /host/crio/scripts/" + docker run --rm --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" + +packer-init: + packer init . + +packer-validate: get-files + packer validate . + +packer-build: packer-init packer-validate + packer build . diff --git a/plugins.pkr.hcl b/plugins.pkr.hcl index f6a58da..61c94f5 100644 --- a/plugins.pkr.hcl +++ b/plugins.pkr.hcl @@ -1,7 +1,7 @@ packer { required_plugins { amazon = { - version = "= 1.1.6" + version = "= 1.2.6" source = "github.com/hashicorp/amazon" } } diff --git a/sysbox-eks.pkr.hcl b/sysbox-eks.pkr.hcl index 97f72cd..d0ed12b 100644 --- a/sysbox-eks.pkr.hcl +++ b/sysbox-eks.pkr.hcl @@ -1,9 +1,9 @@ source "amazon-ebs" "ubuntu-eks" { - ami_name = "${var.img_name}/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/images/hvm-ssd/ubuntu-${var.ubuntu_version}-amd64-server" - ami_description = "Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version}" + ami_name = "${var.img_name}/sysbox-eks_${var.sysbox_version}/k8s_${var.k8s_version}/ubuntu-${var.ubuntu_version}-${var.architecture}-server/${var.img_version}" + ami_description = "Sysbox EKS Node (k8s_${var.k8s_version}), on Ubuntu ${var.ubuntu_version} (${var.architecture}) Maintained by Plural." - region = "us-west-2" - instance_type = "t2.micro" + region = "us-east-2" + instance_type = local.instance_type ami_regions = var.aws_target_regions tags = { @@ -30,7 +30,11 @@ source "amazon-ebs" "ubuntu-eks" { } ssh_username = "ubuntu" - # ami_groups = ["all"] # TODO: uncomment when ready to make public + ami_groups = ["all"] force_deregister = true force_delete_snapshot = true } + +locals { + instance_type = var.architecture == "amd64" ? "t3.micro" : "t4g.micro" +} diff --git a/variables.pkr.hcl b/variables.pkr.hcl index eff06e7..059661d 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -1,11 +1,34 @@ variable "aws_target_regions" { type = list(string) - default = ["us-east-1", "us-east-2", "us-west-2", "ap-southeast-2"] + default = [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "ca-central-1", + "eu-central-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "eu-north-1", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-south-1", + "ap-southeast-1", + "ap-southeast-2", + "sa-east-1" + ] } variable "img_name" { type = string - default = "pluraldev-05-cleanup-part5" + default = "plural" +} + +variable "img_version" { + type = string + default = "v0.1.0" } variable "architecture" { From 1c770089fe35e0c7b92bf5f23a5b55ad726d0e53 Mon Sep 17 00:00:00 2001 From: David van der Spek <28541758+DavidSpek@users.noreply.github.com> Date: Fri, 11 Aug 2023 11:16:54 +0200 Subject: [PATCH 18/26] fix: setup semantic release and build caching (#3) Signed-off-by: David van der Spek --- .github/workflows/build.yaml | 85 +++++++++++++++++++++++++++--- .github/workflows/semantic-pr.yaml | 18 +++++++ Makefile | 6 +-- 3 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/semantic-pr.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4a21a92..a844113 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -7,9 +7,53 @@ on: branches: - main jobs: + prepare: + name: Build EKS AMI + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + env: + SYSBOX_VERSION: 0.6.2 + outputs: + new_release_version: ${{ steps.semantic_release.outputs.new_release_version }} + new_release_published: ${{ steps.semantic_release.outputs.new_release_published }} + sysbox_version: ${{ env.SYSBOX_VERSION }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.PLURAL_BOT_PAT }} + - name: 'Setup Node' + uses: actions/setup-node@v3 + if: github.event_name != 'pull_request' + with: + node-version: 18.12.1 + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v3 + id: semantic_release + with: + dry_run: true + env: + GITHUB_TOKEN: ${{ secrets.PLURAL_BOT_PAT }} + NODE_AUTH_TOKEN: ${{ secrets.PLURAL_BOT_NPM_TOKEN }} + - name: Cache sysbox and cri-o files + id: sysbox_cache + uses: actions/cache@v3 + with: + path: tmp + key: ${{ runner.os }}-build-${{ env.SYSBOX_VERSION }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + if: steps.sysbox_cache.outputs.cache-hit != 'true' + - name: Get sysbox and cri-o files + if: steps.sysbox_cache.outputs.cache-hit != 'true' + run: make get-files packer_build_eks: name: Build EKS AMI runs-on: ubuntu-latest + needs: prepare permissions: contents: 'read' id-token: 'write' @@ -18,25 +62,27 @@ jobs: k8s_version: ["1.23", "1.24", "1.25", "1.26"] ubuntu_version: ["focal-20.04"] architecture: ["amd64", "arm64"] + sysbox_version: ["${{ needs.prepare.outputs.sysbox_version }}"] steps: - name: Checkout uses: actions/checkout@v3 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 - if: github.event_name != 'pull_request' + if: always() && (github.event_name != 'pull_request') with: aws-region: us-east-2 role-to-assume: arn:aws:iam::654897662046:role/github-actions/plural-sysbox-amis-packer role-session-name: SysboxAmisPacker - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Get files - run: make get-files - name: Setup `packer` uses: hashicorp/setup-packer@main id: setup with: version: 1.9.2 + - name: Restore downloaded files + uses: actions/cache/restore@v3 + with: + path: tmp + key: ${{ runner.os }}-build-${{ matrix.sysbox_version }} - name: Run `packer init` id: init run: "packer init ." @@ -45,12 +91,39 @@ jobs: run: "packer validate ." - name: Run `packer build` id: build - if: github.event_name != 'pull_request' + if: always() && (github.event_name != 'pull_request' && needs.prepare.outputs.new_release_published == 'true') env: PKR_VAR_k8s_version: ${{ matrix.k8s_version }} PKR_VAR_ubuntu_version: ${{ matrix.ubuntu_version }} PKR_VAR_architecture: ${{ matrix.architecture }} + PKR_VAR_sysbox_version: ${{ matrix.sysbox_version }} + PKR_VAR_img_version: ${{ needs.prepare.outputs.new_release_version }} run: "packer build ." + release: + runs-on: ubuntu-latest + needs: packer_build_eks + permissions: + contents: 'read' + id-token: 'write' + if: github.event_name != 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + token: ${{ secrets.PLURAL_BOT_PAT }} + - name: 'Setup Node' + uses: actions/setup-node@v3 + if: github.event_name != 'pull_request' + with: + node-version: 18.12.1 + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v3 + id: semantic_release + if: github.event_name != 'pull_request' + env: + GITHUB_TOKEN: ${{ secrets.PLURAL_BOT_PAT }} + NODE_AUTH_TOKEN: ${{ secrets.PLURAL_BOT_NPM_TOKEN }} # trivy-scan: # name: Trivy fs scan # runs-on: ubuntu-latest diff --git a/.github/workflows/semantic-pr.yaml b/.github/workflows/semantic-pr.yaml new file mode 100644 index 0000000..cc02fbb --- /dev/null +++ b/.github/workflows/semantic-pr.yaml @@ -0,0 +1,18 @@ +name: "Semantic PR" + +on: + workflow_dispatch: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index 6a12504..ed21d27 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SYSBOX_VERSION=v0.6.2 +SYSBOX_VERSION ?= 0.6.2 get-files: rm -rf ./tmp @@ -6,8 +6,8 @@ get-files: mkdir -p ./tmp/sysbox/arm64/bin mkdir -p ./tmp/crio/amd64 mkdir -p ./tmp/crio/arm64 - docker run --rm --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/ && cp -r /opt/crio-deploy/scripts/ /host/crio/scripts/" - docker run --rm --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" + docker run --rm --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:v${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/ && cp -r /opt/crio-deploy/scripts/ /host/crio/scripts/" + docker run --rm --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:v${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" packer-init: packer init . From b5bd9719d911e168e2dea9bff18255a8083cdf58 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 11 Aug 2023 11:18:18 +0200 Subject: [PATCH 19/26] add semantic release config file Signed-off-by: David van der Spek --- .releaserc | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .releaserc diff --git a/.releaserc b/.releaserc new file mode 100644 index 0000000..ebec692 --- /dev/null +++ b/.releaserc @@ -0,0 +1,5 @@ +branches: ["main"] +plugins: +- "@semantic-release/commit-analyzer" +- "@semantic-release/release-notes-generator" +- "@semantic-release/github" From e9bd09f20533ea41d65901cc3bc88469aa3cddea Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 11 Aug 2023 11:42:58 +0200 Subject: [PATCH 20/26] ci: disable fail fast in the build matrix Signed-off-by: David van der Spek --- .github/workflows/build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a844113..7e1e2ac 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -58,6 +58,7 @@ jobs: contents: 'read' id-token: 'write' strategy: + fail-fast: false matrix: k8s_version: ["1.23", "1.24", "1.25", "1.26"] ubuntu_version: ["focal-20.04"] @@ -68,7 +69,7 @@ jobs: uses: actions/checkout@v3 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 - if: always() && (github.event_name != 'pull_request') + if: github.event_name != 'pull_request' with: aws-region: us-east-2 role-to-assume: arn:aws:iam::654897662046:role/github-actions/plural-sysbox-amis-packer From 3c92516b4b7f9045fc2cc3aa1ba099d097b719b5 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 11 Aug 2023 12:32:19 +0200 Subject: [PATCH 21/26] disable regions with quota issue Signed-off-by: David van der Spek --- .github/workflows/build.yaml | 3 ++- variables.pkr.hcl | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7e1e2ac..79c7abf 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,7 +68,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v2 if: github.event_name != 'pull_request' with: aws-region: us-east-2 @@ -92,6 +92,7 @@ jobs: run: "packer validate ." - name: Run `packer build` id: build + # always is used here to ensure the builds can't get cancelled and leave dangling resources if: always() && (github.event_name != 'pull_request' && needs.prepare.outputs.new_release_published == 'true') env: PKR_VAR_k8s_version: ${{ matrix.k8s_version }} diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 059661d..1ea1129 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -4,7 +4,7 @@ variable "aws_target_regions" { "us-east-1", "us-east-2", "us-west-1", - "us-west-2", + # "us-west-2", "ca-central-1", "eu-central-1", "eu-west-1", @@ -15,7 +15,7 @@ variable "aws_target_regions" { "ap-northeast-2", "ap-northeast-3", "ap-south-1", - "ap-southeast-1", + # "ap-southeast-1", "ap-southeast-2", "sa-east-1" ] From 2f562cdc355f91789434a10d311c30b089ee6c30 Mon Sep 17 00:00:00 2001 From: "plural-renovate[bot]" <117748337+plural-renovate[bot]@users.noreply.github.com> Date: Fri, 11 Aug 2023 13:06:26 +0200 Subject: [PATCH 22/26] Configure Renovate (#1) Co-authored-by: plural-renovate[bot] --- renovate.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..9977019 --- /dev/null +++ b/renovate.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:base", + ":dependencyDashboard", + ":prHourlyLimit4", + ":semanticCommits", + ":prConcurrentLimit10" + ], + "packageRules": [ + { + "matchDatasources": [ + "docker" + ], + "matchUpdateTypes": [ + "major" + ], + "enabled": true + } + ], + "labels": [ + "dependencies" + ], + "separateMinorPatch": true +} From 11ffc0537a9e958e1701fff98ad25fa8ba1a6561 Mon Sep 17 00:00:00 2001 From: David van der Spek Date: Fri, 11 Aug 2023 13:47:58 +0200 Subject: [PATCH 23/26] ci: setup renovate to manage sysbox version Signed-off-by: David van der Spek --- .github/workflows/build.yaml | 3 ++- Makefile | 6 +++--- renovate.json | 9 +++++++++ variables.pkr.hcl | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 79c7abf..b5cab34 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,7 +14,8 @@ jobs: contents: 'read' id-token: 'write' env: - SYSBOX_VERSION: 0.6.2 + # renovate: datasource=github-tags depName=nestybox/sysbox + SYSBOX_VERSION: v0.6.2 outputs: new_release_version: ${{ steps.semantic_release.outputs.new_release_version }} new_release_published: ${{ steps.semantic_release.outputs.new_release_published }} diff --git a/Makefile b/Makefile index ed21d27..7fbcff1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SYSBOX_VERSION ?= 0.6.2 +SYSBOX_VERSION ?= v0.6.2 get-files: rm -rf ./tmp @@ -6,8 +6,8 @@ get-files: mkdir -p ./tmp/sysbox/arm64/bin mkdir -p ./tmp/crio/amd64 mkdir -p ./tmp/crio/arm64 - docker run --rm --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:v${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/ && cp -r /opt/crio-deploy/scripts/ /host/crio/scripts/" - docker run --rm --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:v${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" + docker run --rm --platform linux/amd64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/amd64/bin/ && cp -r /opt/sysbox/systemd/ /host/sysbox/systemd/ && cp -r /opt/crio-deploy/bin/* /host/crio/amd64/ && cp -r /opt/crio-deploy/config/ /host/crio/config/ && cp -r /opt/crio-deploy/scripts/ /host/crio/scripts/" + docker run --rm --platform linux/arm64 -v ./tmp:/host registry.nestybox.com/nestybox/sysbox-deploy-k8s:${SYSBOX_VERSION} /bin/bash -c "cp /opt/sysbox/bin/generic/* /host/sysbox/arm64/bin/ && cp -r /opt/crio-deploy/bin/* /host/crio/arm64/" packer-init: packer init . diff --git a/renovate.json b/renovate.json index 9977019..4731a4a 100644 --- a/renovate.json +++ b/renovate.json @@ -18,6 +18,15 @@ "enabled": true } ], + "regexManagers": [ + { + "fileMatch": ["^.github/workflows/build.yaml$"], + "matchStrings": [ + "datasource=(?.*?) depName=(?.*?)( versioning=(?.*?))?\\s .*?_VERSION: (?.*)" + ], + "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}" + } + ], "labels": [ "dependencies" ], diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 1ea1129..e54e346 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -48,10 +48,10 @@ variable "ubuntu_version" { variable "sysbox_version" { type = string - default = "0.6.2" + default = "v0.6.2" validation { - condition = can(regex("^\\d+\\.\\d+\\.\\d+$", var.sysbox_version)) + condition = can(regex("^v?\\d+\\.\\d+\\.\\d+$", var.sysbox_version)) error_message = "Invalid Sysbox version: expected '{major}.{minor}.{patch}'." } } From 21adea054b9ed61d8fa4a7be141aa21d2324a9ca Mon Sep 17 00:00:00 2001 From: "plural-renovate[bot]" <117748337+plural-renovate[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:19:24 +0200 Subject: [PATCH 24/26] chore(deps): update docker/setup-qemu-action action to v3 (#8) Co-authored-by: plural-renovate[bot] --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b5cab34..79f2a1d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -46,7 +46,7 @@ jobs: path: tmp key: ${{ runner.os }}-build-${{ env.SYSBOX_VERSION }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 if: steps.sysbox_cache.outputs.cache-hit != 'true' - name: Get sysbox and cri-o files if: steps.sysbox_cache.outputs.cache-hit != 'true' From 3d6d38e790569e036959da0c9aa41e3f378042cc Mon Sep 17 00:00:00 2001 From: "plural-renovate[bot]" <117748337+plural-renovate[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:19:35 +0200 Subject: [PATCH 25/26] chore(deps): update aws-actions/configure-aws-credentials action to v4 (#7) Co-authored-by: plural-renovate[bot] --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 79f2a1d..bc29381 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -69,7 +69,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 if: github.event_name != 'pull_request' with: aws-region: us-east-2 From 0ed0fcaa4235c370dd4528df701602b483a84e21 Mon Sep 17 00:00:00 2001 From: "plural-renovate[bot]" <117748337+plural-renovate[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:19:43 +0200 Subject: [PATCH 26/26] chore(deps): update actions/checkout action to v4 (#6) Co-authored-by: plural-renovate[bot] --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bc29381..2e1ca98 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -22,7 +22,7 @@ jobs: sysbox_version: ${{ env.SYSBOX_VERSION }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.PLURAL_BOT_PAT }} @@ -67,7 +67,7 @@ jobs: sysbox_version: ["${{ needs.prepare.outputs.sysbox_version }}"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 if: github.event_name != 'pull_request' @@ -111,7 +111,7 @@ jobs: if: github.event_name != 'pull_request' steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.PLURAL_BOT_PAT }}