From 0f8c2af4809cbf223fce6254fc7cd2b81bffe92f Mon Sep 17 00:00:00 2001 From: John McBride Date: Thu, 27 Oct 2022 17:52:02 +0000 Subject: [PATCH] v1.0.0 release Signed-off-by: John McBride --- .env | 2 +- CHANGELOG.md | 56 +++ README.md | 123 +++--- VERSION | 2 +- design/1.0.0-release.md | 395 ++++++++++++++++++ .../deploy/bottlerocket-update-operator.yaml | 6 +- 6 files changed, 523 insertions(+), 61 deletions(-) create mode 100644 design/1.0.0-release.md diff --git a/.env b/.env index b37dc2bc..0e881fa8 100644 --- a/.env +++ b/.env @@ -1,6 +1,6 @@ # Example .env file -BRUPOP_CONTAINER_IMAGE=public.ecr.aws/bottlerocket/bottlerocket-update-operator:v0.2.2 +BRUPOP_CONTAINER_IMAGE=public.ecr.aws/bottlerocket/bottlerocket-update-operator:v1.0.0 # If testing against a private image registry, you can set the pull secret to fetch images. # This can likely remain as `brupop` so long as you run something like the following: diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d74b71..e4e5c87b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,59 @@ +# 1.0.0 + +## General + +Added + +* Mechanism to constrain updates to a certain update time window ([#241]) +* Option to exclude node before draining - ([#231]) +* Port configuration ([#315]) +* Support for concurrent updates - ([#238]) +* Automatic prometheus scraping annotations for controller's service ([#269]) +* Use `ca.crt` in SSL - ([#260]) +* Reload certificates periodically to ensure no service loss ([#280]) +* Replaced `bunyan` style logging in favor of human readable logs ([#298]) +* Support webhook conversions from v2 to v1 (to support the Kubernetes pinwheel model) ([#308]) +* Support integration tests in AWS China region ([#317]) ([#318]) + +Fixed + +* Upgraded Bottlerocket SDK to consume fix for OpenSSl CVE-2022-3602 and CVE-2022-3786 ([#331]) +* Gracefully exit Brupop agent when rebooting node ([#218]) +* Clean up `bottlerocketshadows` when Brupop resources are removed from the cluster ([#235]) +* Clarify crossbeam license ([#250]) +* Made error handling module specific ([#279]) ([#291]) + +Misc + +* Numerous dependency updates +* Fixed clippy linting / warnings ([#267]) +* Clear and remove GitHub actions cache ([#268]) ([#286]) +* Added step to integration tests to automatically add and delete cert-manager ([#320]) +* Added GitHub action step to catch changes to deployment manifest ([#321]) + +[#218]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/218 +[#231]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/231 +[#235]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/235 +[#238]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/238 +[#241]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/241 +[#250]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/250 +[#260]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/260 +[#267]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/267 +[#268]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/268 +[#269]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/269 +[#279]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/279 +[#280]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/280 +[#286]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/286 +[#291]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/291 +[#298]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/298 +[#308]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/308 +[#315]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/315 +[#317]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/317 +[#318]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/318 +[#320]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/320 +[#321]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/321 +[#331]: https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/331 + # 0.2.2 ## General diff --git a/README.md b/README.md index 79c0152c..e7316752 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,27 @@ # Bottlerocket Update Operator -The Bottlerocket update operator is a [Kubernetes operator](https://Kubernetes.io/docs/concepts/extend-Kubernetes/operator/) that coordinates Bottlerocket updates on hosts in a cluster. +The Bottlerocket update operator (or, for short, Brupop) is a [Kubernetes operator](https://Kubernetes.io/docs/concepts/extend-Kubernetes/operator/) that coordinates Bottlerocket updates on hosts in a cluster. When installed, the Bottlerocket update operator starts a controller deployment on one node, an agent daemon set on every Bottlerocket node, and an Update Operator API Server deployment. The controller orchestrates updates across your cluster, while the agent is responsible for periodically querying for Bottlerocket updates, draining the node, and performing the update when asked by the controller. The agent performs all cluster object mutation operations via the API Server, which performs additional authorization using the Kubernetes TokenReview API -- ensuring that any request associated with a node is being made by the agent pod running on that node. +Further, `cert-manager` is required in order for the API server to use a CA certificate to communicate over SSL with the agents. Updates to Bottlerocket are rolled out in [waves](https://github.com/bottlerocket-os/bottlerocket/tree/develop/sources/updater/waves) to reduce the impact of issues; the nodes in your cluster may not all see updates at the same time. +For a deep dive on installing Brupop, how it works, and it's integration with Bottlerocket, [check out this design deep dive document!](./design/1.0.0-release.md) + ## Getting Started ### Installation -We use [cert-manager](https://cert-manager.io) to manage self-signed certificates used by Bottlerocket update operator. To install cert-manager: +1. Brupop uses [cert-manager](https://cert-manager.io) to manage self-signed certificates used by the Bottlerocket update operator. To install cert-manager: ```sh -kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.2/cert-manager.yaml +kubectl apply -f \ + https://github.com/cert-manager/cert-manager/releases/download/v1.8.2/cert-manager.yaml ``` -We can install the Bottlerocket update operator using the recommended configuration defined in [bottlerocket-update-operator.yaml](https://github.com/bottlerocket-os/bottlerocket-update-operator/blob/0.2.2/yamlgen/deploy/bottlerocket-update-operator.yaml): +2. The Bottlerocket update operator can then be installed using the recommended configuration defined in [bottlerocket-update-operator.yaml](https://github.com/bottlerocket-os/bottlerocket-update-operator/blob/1.0.0/yamlgen/deploy/bottlerocket-update-operator.yaml). +This YAML file can also be found in the [Brupop release artifacts](https://github.com/bottlerocket-os/bottlerocket-update-operator/releases): ```sh kubectl apply -f ./bottlerocket-update-operator.yaml @@ -24,12 +29,19 @@ kubectl apply -f ./bottlerocket-update-operator.yaml This will create the required namespace, custom resource definition, roles, deployments, etc., and use the latest update operator image available in [Amazon ECR Public](https://gallery.ecr.aws/bottlerocket/bottlerocket-update-operator). -Note: The above link is set to use the configuration for the latest version of the Update Operator, `v0.2.2`. +Note: The above link points to the latest version of the Update Operator, `v1.0.0`. Be sure to use the git tag for the Update Operator release that you plan to deploy. -#### Configuration +3. Label nodes with `bottlerocket.aws/updater-interface-version=2.0.0` to indicate they should be automatically updated. +Only bottlerocket nodes with this label will be updated. For more information on labeling, refer to the [Label nodes section of this readme.](https://github.com/bottlerocket-os/bottlerocket-update-operator#label-nodes) + +```sh +kubectl label node MY_NODE_NAME bottlerocket.aws/updater-interface-version=2.0.0 +``` + +### Configuration -##### Configure API server ports +#### Configure API server ports If you'd like to configure what ports the API server uses, adjust the value that is consumed in the container environment: @@ -42,10 +54,9 @@ adjust the value that is consumed in the container environment: env: - name: APISERVER_INTERNAL_PORT value: 999 - ... ``` -You'll then also need to adjust the various "port" entries in the manifest +You'll then also need to adjust the various "port" entries in the YAML manifest to correctly reflect what port the API server starts on and expects for its service port: ```yaml @@ -59,18 +70,18 @@ to correctly reflect what port the API server starts on and expects for its serv port: 123 ``` -The default values are generated from the `.env` file (and can be used when building the update operator from source to configure your own ports): +The default values are generated from the [`.env`](https://github.com/bottlerocket-os/bottlerocket-update-operator/blob/develop/.env) file (and can be used when building the update operator from source to configure your own ports): - `APISERVER_INTERNAL_PORT = 8443` - This is the container port the API server starts on. If this environment variable is _not_ found, the Brupop API server will fail to start. - `APISERVER_SERVICE_PORT = 443` - This is the port Brupop's Kubernetes Service uses to target the internal API Server port. It is used by the node agents to access the API server. If this environment variable is _not_ found, the Brupop agents will fail to start. -##### Exclude Nodes from Load Balancers Before Draining -This configuration uses Kuberenetes [ServiceNodeExclusion](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) feature. +#### Exclude Nodes from Load Balancers Before Draining +This configuration uses the Kuberenetes [ServiceNodeExclusion](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) feature. `EXCLUDE_FROM_LB_WAIT_TIME_IN_SEC` can be used to enable the feature to exclude the node from load balancer before draining. When `EXCLUDE_FROM_LB_WAIT_TIME_IN_SEC` is 0 (default), the feature is disabled. -When `EXCLUDE_FROM_LB_WAIT_TIME_IN_SEC` is set to a positive integer, bottlerocket update operater will exclude the node from +When `EXCLUDE_FROM_LB_WAIT_TIME_IN_SEC` is set to a positive integer, bottlerocket update operator will exclude the node from load balancer and then wait `EXCLUDE_FROM_LB_WAIT_TIME_IN_SEC` seconds before draining the pods on the node. To enable this feature, go to `bottlerocket-update-operator.yaml`, change `EXCLUDE_FROM_LB_WAIT_TIME_IN_SEC` to a positive integer value. @@ -90,16 +101,16 @@ For example: ... ``` -##### Set Up Max Concurrent Update +#### Set Up Max Concurrent Update `MAX_CONCURRENT_UPDATE` can be used to specify the max concurrent updates during updating. -When `MAX_CONCURRENT_UPDATE` is a positive integer number, bottlerocket update operator +When `MAX_CONCURRENT_UPDATE` is a positive integer, the bottlerocket update operator will concurrently update up to `MAX_CONCURRENT_UPDATE` nodes respecting [PodDisruptionBudgets](https://kubernetes.io/docs/tasks/run-application/configure-pdb/). When `MAX_CONCURRENT_UPDATE` is set to `unlimited`, bottlerocket update operator will concurrently update all nodes respecting [PodDisruptionBudgets](https://kubernetes.io/docs/tasks/run-application/configure-pdb/). Note: The `MAX_CONCURRENT_UPDATE` configuration does not work well with `EXCLUDE_FROM_LB_WAIT_TIME_IN_SEC` configuration, especially when `MAX_CONCURRENT_UPDATE` is set to `unlimited`, it could potentially exclude all -nodes from load balancer at the same time.. +nodes from load balancer at the same time. To enable this feature, go to `bottlerocket-update-operator.yaml`, change `MAX_CONCURRENT_UPDATE` to a positive integer value or `unlimited`. For example: @@ -116,11 +127,11 @@ For example: value: "1" ``` -##### Set Up Update Time Window +#### Set an Update Time Window `UPDATE_WINDOW_START` and `UPDATE_WINDOW_STOP` can be used to specify the time window in which updates are permitted. -When `UPDATE_WINDOW_START` and `UPDATE_WINDOW_STOP` is 0:0:0 (default), the feature is disabled. +When `UPDATE_WINDOW_START` and `UPDATE_WINDOW_STOP` are both 0:0:0 (default), the feature is disabled. -To enable this feature, go to `bottlerocket-update-operator.yaml`, change `UPDATE_WINDOW_START` and `UPDATE_WINDOW__STOP` to a `hour:minute:second` formate value (UTC (24-hour time notation)). Note that `UPDATE_WINDOW_START` is inclusive and `UPDATE_WINDOW_STOP` is exclusive. +To enable this feature, go to `bottlerocket-update-operator.yaml`, change `UPDATE_WINDOW_START` and `UPDATE_WINDOW_STOP` to a `hour:minute:second` formatted value (UTC (24-hour time notation)). Note that `UPDATE_WINDOW_START` is inclusive and `UPDATE_WINDOW_STOP` is exclusive. To avoid stopping an in-process node update when the update window stops, Bottlerocket update operator reserves 6 mins before `UPDATE_WINDOW_STOP` to finish remaining updates. @@ -144,7 +155,7 @@ For example: ### Label nodes By default, each Workload resource constrains scheduling of the update operator by limiting pods to Bottlerocket nodes based on their labels. -These labels are not applied on nodes automatically and will need to be set on each using `kubectl`. +These labels are not applied on nodes automatically and will need to be set on each desired node using `kubectl`. The agent relies on each node's updater components and schedules its pods based on their interface supported. The node indicates its updater interface version in a label called `bottlerocket.aws/updater-interface-version`. Agent deployments, respective to the interface version, are scheduled using this label and target only a single version in each. @@ -178,17 +189,15 @@ If all nodes in the cluster are running Bottlerocket and require the same `updat kubectl label node $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') bottlerocket.aws/updater-interface-version=2.0.0 ``` -If you must support `updater-interface-version` 1.0.0, please [open an issue](https://github.com/bottlerocket-os/bottlerocket-update-operator/issues/new/choose) and tell us about your use case. +### Uninstalling -### A Note About Removing Labels +If you remove the `bottlerocket.aws/updater-interface-version=2.0.0` label from a node, +the Brupop controller will remove the resources from that node (including the `bottlerocketshadow` CRD associated with that node). -Should you decide that the update operator should no longer manage a node, removing the `updater-interface-version` is not quite sufficient to remove the update operator components responsible for that node. -The update operator associates a Kubernetes Custom Resource with each node. While the Custom Resource will be garbage collected if the node itself is deleted, you must manually clean up the Custom Resource if you choose to delete only the `updater-interface-version`. The Custom Resource can be deleted like so: +Deleting the custom resources, definitions, and deployments will then fully remove the bottlerocket update operator from your cluster: ```sh -# Set this to the name of the node you wish to stop managing with the update operator. -NODE_NAME="my-node-name" -kubectl delete brs brs-${NODE_NAME} --namespace brupop-bottlerocket-aws +kubectl delete -f ./bottlerocket-update-operator.yaml ``` ## Operation @@ -209,7 +218,7 @@ Additionally, the update operator's controller and apiserver components expose m The current state of the cluster from the perspective of the update operator can be summarized by querying the Kubernetes API for BottlerocketShadow objects. This view will inform you of the current Bottlerocket version of each node managed by the update operator, as well as the current ongoing update status of any node with an update in-progress. -The following command requires `kubectl` to be configured for the development cluster to be monitored: +The following command requires `kubectl` to be configured for the desired cluster to be monitored: ``` sh kubectl get bottlerocketshadows --namespace brupop-bottlerocket-aws @@ -232,7 +241,7 @@ brs-node-2 Idle 1.5.1 StagedUpdat #### Monitoring Cluster History and Metrics with Prometheus -The update operator provides metrics endpoints which can be scraped into [Prometheus](https://prometheus.io/). +The update operator provides metrics endpoints which can be scraped by [Prometheus](https://prometheus.io/). This allows you to monitor the history of update operations using popular metrics analysis and visualization tools. We provide a [sample configuration](./yamlgen/telemetry/prometheus-resources.yaml) which demonstrates a Prometheus deployment into the cluster that is configured to gather metrics data from the update operator. @@ -268,46 +277,48 @@ Search for: `bottlerocket-update-operator.yaml` pulls operator images from Amazon ECR Public. You may also choose to pull from regional Amazon ECR repositories such as the following. - - 917644944286.dkr.ecr.af-south-1.amazonaws.com - - 375569722642.dkr.ecr.ap-east-1.amazonaws.com - - 328549459982.dkr.ecr.ap-northeast-1.amazonaws.com - - 328549459982.dkr.ecr.ap-northeast-2.amazonaws.com - - 328549459982.dkr.ecr.ap-south-1.amazonaws.com - - 328549459982.dkr.ecr.ap-southeast-1.amazonaws.com - - 328549459982.dkr.ecr.ap-southeast-2.amazonaws.com - - 328549459982.dkr.ecr.ca-central-1.amazonaws.com - - 328549459982.dkr.ecr.eu-central-1.amazonaws.com - - 328549459982.dkr.ecr.eu-north-1.amazonaws.com - - 586180183710.dkr.ecr.eu-south-1.amazonaws.com - - 328549459982.dkr.ecr.eu-west-1.amazonaws.com - - 328549459982.dkr.ecr.eu-west-2.amazonaws.com - - 328549459982.dkr.ecr.eu-west-3.amazonaws.com - - 509306038620.dkr.ecr.me-south-1.amazonaws.com - - 328549459982.dkr.ecr.sa-east-1.amazonaws.com - - 328549459982.dkr.ecr.us-east-1.amazonaws.com - - 328549459982.dkr.ecr.us-east-2.amazonaws.com - - 328549459982.dkr.ecr.us-west-1.amazonaws.com - - 328549459982.dkr.ecr.us-west-2.amazonaws.com - - 388230364387.dkr.ecr.us-gov-east-1.amazonaws.com - - 347163068887.dkr.ecr.us-gov-west-1.amazonaws.com + - `917644944286.dkr.ecr.af-south-1.amazonaws.com` + - `375569722642.dkr.ecr.ap-east-1.amazonaws.com` + - `328549459982.dkr.ecr.ap-northeast-1.amazonaws.com` + - `328549459982.dkr.ecr.ap-northeast-2.amazonaws.com` + - `328549459982.dkr.ecr.ap-northeast-3.amazonaws.com` + - `328549459982.dkr.ecr.ap-south-1.amazonaws.com` + - `328549459982.dkr.ecr.ap-southeast-1.amazonaws.com` + - `328549459982.dkr.ecr.ap-southeast-2.amazonaws.com` + - `386774335080.dkr.ecr.ap-southeast-3.amazonaws.com` + - `328549459982.dkr.ecr.ca-central-1.amazonaws.com` + - `328549459982.dkr.ecr.eu-central-1.amazonaws.com` + - `328549459982.dkr.ecr.eu-north-1.amazonaws.com` + - `586180183710.dkr.ecr.eu-south-1.amazonaws.com` + - `328549459982.dkr.ecr.eu-west-1.amazonaws.com` + - `328549459982.dkr.ecr.eu-west-2.amazonaws.com` + - `328549459982.dkr.ecr.eu-west-3.amazonaws.com` + - `553577323255.dkr.ecr.me-central-1.amazonaws.com` + - `509306038620.dkr.ecr.me-south-1.amazonaws.com` + - `328549459982.dkr.ecr.sa-east-1.amazonaws.com` + - `328549459982.dkr.ecr.us-east-1.amazonaws.com` + - `328549459982.dkr.ecr.us-east-2.amazonaws.com` + - `328549459982.dkr.ecr.us-west-1.amazonaws.com` + - `328549459982.dkr.ecr.us-west-2.amazonaws.com` + - `388230364387.dkr.ecr.us-gov-east-1.amazonaws.com` + - `347163068887.dkr.ecr.us-gov-west-1.amazonaws.com` + - `183470599484.dkr.ecr.cn-north-1.amazonaws.com.cn` + - `183901325759.dkr.ecr.cn-northwest-1.amazonaws.com.cn` Example regional image URI: ``` -328549459982.dkr.ecr.us-west-2.amazonaws.com/bottlerocket-update-operator:v0.2.2 +328549459982.dkr.ecr.us-west-2.amazonaws.com/bottlerocket-update-operator:v1.0.0 ``` ### Current Limitations -- Communication between the bottlerocket agents and API server does not currently use SSL, due to a requirement for a cert management solution. - We are considering an approach which uses [cert-manager](https://cert-manager.io) to that end. - Monitoring on newly-rebooted nodes is limited. We are considering an approach in which custom health checks can be configured to run after reboots. (https://github.com/bottlerocket-os/bottlerocket/issues/503) -- single node cluster degrades into unscheduleable on update (https://github.com/bottlerocket-os/bottlerocket/issues/501) - Node labels are not automatically applied to allow scheduling (https://github.com/bottlerocket-os/bottlerocket/issues/504) ## Troubleshooting -When installed with the [default deployment](https://github.com/bottlerocket-os/bottlerocket-update-operator/blob/v0.2.2/yamlgen/deploy/bottlerocket-update-operator.yaml), the logs can be fetched through Kubernetes deployment logs. +When installed with the [default deployment](https://github.com/bottlerocket-os/bottlerocket-update-operator/blob/v1.0.0/yamlgen/deploy/bottlerocket-update-operator.yaml), the logs can be fetched through Kubernetes deployment logs. Because mutations to a node are orchestrated through the API server component, searching those deployment logs for a node ID can be useful. To get logs for the API server, run the following: diff --git a/VERSION b/VERSION index 1474d00f..0ec25f75 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.2.0 +v1.0.0 diff --git a/design/1.0.0-release.md b/design/1.0.0-release.md new file mode 100644 index 00000000..0018d5d1 --- /dev/null +++ b/design/1.0.0-release.md @@ -0,0 +1,395 @@ +# Bottlerocket-update-operator Deep Dive + +*Authors*: Tianhao Geng (@gthao313) & John McBride (@jpmcb) + +Bottlerocket-update-operator (affectionately nick-named “Brupop”) is a dedicated +kubernetes controller suite for keeping your bottlerocket hosts up to date with +the latest releases! Now, Kubernetes operators who manage clusters with +Bottlerocket nodes can be confident that they are getting the latest releases +with the latest security upgrades and newest features. + +## Installation + +Before we can install the update operator, we need to install cert-manager to +the cluster: + +```sh +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.8.2/cert-manager.yaml +``` + +Cert-manager is needed to generate certificates that are used by the Brupop API +server to ensure secure and trusted connections are being made from individual +upgrade agents. + +We can then install the Bottlerocket update operator using the recommended +configuration defined in the yaml manifest: + +```sh +kubectl apply -f ./bottlerocket-update-operator.yaml +``` + +This yaml file can be found at the root of [the code repository](https://github.com/bottlerocket-os/bottlerocket-update-operator/blob/develop/bottlerocket-update-operator.yaml) +or as [a static file in the newest release](https://github.com/bottlerocket-os/bottlerocket-update-operator/releases). +This will create the required namespace, custom resource definition, roles, +deployments, service accounts, etc. + +## Label nodes + +By default, the update operator is constrained to limit its pods to Bottlerocket +nodes based on their labels. This way, Kubernetes operators can declaratively +define the Bottlerocket nodes that exist for a cluster. These labels are not +applied on nodes automatically and will need to be set on each Bottlerocket +node using kubectl. The agent relies on each node's updater components and +schedules its pods based on their supported interface. The node indicates its +updater interface version in a label called bottlerocket.aws/updater-interface-version + +The most current version of the updater interface is 2.0.0. Here is the sample +command to label a single node: + +```sh +kubectl label node NODE_NAME bottlerocket.aws/updater-interface-version=2.0.0 +``` + +If all nodes in the cluster are running Bottlerocket, you can label all at the +same time by running the following command: + +```sh +kubectl label node $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') bottlerocket.aws/updater-interface-version=2.0.0 +``` + +## Observing state + +The current state of the cluster from the perspective of the update operator +can be summarized by querying the Kubernetes API for BottlerocketShadow +objects, the Custom Resource for the update operator. This view will inform +you of the current Bottlerocket version of each node managed by the update +operator, as well as the current ongoing update status of any node with an +update in-progress. + +# Deep dive + +Before we can get too deep into the update operator, let’s explore what makes +bottlerocket a unique solution for container runtimes like Kubernetes. +Bottlerocket is a Linux-based open-source operating system that is +purpose-built by Amazon Web Services for running containers. Bottlerocket +focuses on security and maintainability, providing a reliable, consistent, and +safe platform for container-based workloads. This is a reflection of what we've +learned building operating systems and services at Amazon. Bottlerocket +includes only the essential software required to run containers, and is built +with standard open-source components. Bottlerocket specific additions focus on +reliable updates and on the API. Instead of making configuration changes +manually, you can change settings with an API call, and these changes are +automatically migrated through updates. + +Some notable security features include: + +## Immutable rootfs backed by dm-verity + +Bottlerocket uses [dm-verity](https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html) +for its root filesystem image. This provides transparent integrity checking of +the underlying block device using a cryptographic digest. The root filesystem +is marked as read-only and cannot be directly modified by userspace processes. +The kernel is configured to restart if corruption is detected. That allows the +system to fail fast if the underlying block device is unexpectedly modified. + +## Stateless `tmpfs` for `/etc` + +Bottlerocket uses [`tmpfs`](https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt), +a memory-backed filesystem, for `/etc`. + +Direct modification of system configuration files such as `/etc/resolv.conf` +is not supported. This makes OS updates more reliable, as it is not necessary +to account for local edits that might have changed the behavior of system +services. It also makes it harder for an attacker to modify these files in a +way that persists across a reboot. + +## SELinux enabled in enforcing mode + +Bottlerocket runs [SELinux](http://www.selinuxproject.org/page/Main_Page) +in enforcing mode by default and does not allow users to disable it. + +SELinux is a Linux Security Module (LSM) that provides a mechanism for mandatory +access control (MAC). Processes that run as root with full capabilities are +still subject to the mandatory policy restrictions. + +The policy in Bottlerocket has the following objectives: + +1. Prevent most components from directly modifying the API settings. +2. Block most components from modifying the container archives saved on disk. +3. Stop containers from directly modifying the layers for other running containers. + +The policy is currently aimed at hardening the OS against persistent threats. +Continued enhancements to the policy will focus on mitigating the impact of +OS vulnerabilities + +## No shell or interpreters installed + +Bottlerocket does not have a shell and interpreted languages such as Python +are not available as packages. + +Shells and interpreters enable administrators to write code that combines other +programs on the system in new ways. However, these properties can also be +exploited by an attacker to pivot from a vulnerability that grants local +code execution. The lack of a shell also serves as a forcing function to ensure +that new code for the OS is written in a preferred language such as Rust or Go. +These languages offer built-in protection against memory safety issues such as +buffer overflows. + +## Automated security updates + +Bottlerocket is designed for reliable security updates that can be applied +through automation (such as Brupop). + +This is achieved through the following mechanisms: + +* Two partition sets and an active/passive flip to swap OS images, You + might know of this as A/B (Seamless) OS updates +* Declarative API with modeled settings for runtime configuration +* Variants to silo backwards-incompatible or breaking changes + +Using partition sets and modeled settings removes the dependency on correct +local state for reliable updates. There is no package manager database or +shared filesystem tree that can become corrupted and make the process +non-deterministic. + +Our philosophy is that the right time for an unexpected major version update +to the kernel or orchestrator agent is "never." For this reason, any major +updates to these parts are included in a new variant of Bottlerocket. +Variant crossgrading is not currently supported. + +# A glance at the Bottlerocket API system + +Bottlerocket is intended to be an API-first operating system - direct user +interaction with Bottlerocket is usually through the API. The API server +handles requests, the remaining components make sure the system is up to date, +and that requests are persisted and applied correctly. As opposed to more +traditional Operating Systems which rely on modifying configuration files, the +API system is the bridge between the user and the underlying system. It aims to +simplify common configuration, improve reliability, and reduce the need for the +user to log in and debug. + +Data store is a key/value store that serves as the central storage location for +the API system and tools using the API. + +## The update API + +Users can access the API through the apiclient binary available in Bottlerocket +which may be accessed through a control-channel like SSM or the admin container. +Rust code can also use the apiclient library to make requests to the Unix-domain +socket of the apiserver. + +An important feature that Brupop uses here is the Update API. The Bottlerocket API +includes methods for checking and starting system updates. apiclient knows how to +handle those update APIs for you. Let’s look at a few common uses of the apiclient +when applying updates: + +### To see what updates are available + +```sh +apiclient update check +``` + +If an update is available, it will show up in the `chosen_update` field. +The available_updates field will show the full list of available versions, +including older versions, because Bottlerocket supports safely rolling back. + +### To apply the latest update + +```sh +apiclient update apply +``` + +The next time you reboot, you'll start up in the new version, and system +configuration will be automatically migrated. + +### To reboot right away + +```sh +apiclient reboot +``` + +The system will automatically roll back if it's unable to boot. + +Brupop uses many of these mechanisms to apply updates automatically to +Bottlerocket nodes in your cluster. Now, let’s take a closer look at Brupop +itself. + +# The Update Operator Design + +The Bottlerocket update operator is a Kubernetes operator that coordinates +Bottlerocket updates on hosts in a kubernetes cluster. + +Brupop aims to: + +* Control the rate at which updates are deployed — Since bottlerocket requires + reboot for update, the workload of the updated hosts will be distributed on the + rest of the cluster, we don't want to overload the cluster and constrain + resources. +* Safely abort any work being performed on Nodes before they are updated — Before + any node is updated, the system tells Kubernetes not to schedule additional + worker Pods there. Then we safely remove existing pods using an operation + that we call a "drain". +* Prevent dangerous updates from spreading across all hosts by catching + errors — If something bad happens during the update, Brupop backs off + attempting to continue and prevents potentially bricking the entire cluster. + +Keep in mind that the end goal is to safely use the Bottlerocket API on each of +our hosts to perform updates just like if we were using the apiclient through +the admin container on an individual host. + +## System overview and architecture + +Brupop consists of three primary components: the controller, the agent, and the +apiserver. + +The controller orchestrates updates across your cluster, while the agent is +responsible for periodically querying for Bottlerocket updates, draining the +node, and performing the update when asked by the controller. The agent +performs all cluster object mutation operations via the Brupop API Server, +which performs additional authorization using the Kubernetes TokenReview API +and certificates generated by Cert-Manager — ensuring that any request +associated with a node is being made by the agent pod running on that node. + +When installed, the Bottlerocket update operator starts a controller deployment +on one node, an agent daemon set on every Bottlerocket node, and an Update +Operator API Server deployment. All components together collaborate to drive +each Bottlerocket node through an update when updates are available. + +## K8s controllers, in a nut shell + +To understand Brupop’s design, one must understand the Kubernetes Controller +pattern (http://(https//kubernetes.io/docs/concepts/architecture/controller/). +State in Kubernetes is managed by a “Controller” which executes a control loop. +A control loop has a target desired state, and a measured current state (like a +thermostat, with a set temperature and a current temperature, You change the +temperature you desire on your thermostat (which is the spec), and the HVAC +system works in a loop to bring your house's temperature (the status) to the +one you configured.). The controller performs the actions necessary to drive +the current state towards the desired state. + +In Kubernetes, every abstraction is stored in the backend as an object which has +a spec representing the desired state of the object, as well as a status +representing the current state. Controllers use these objects to perform the +features expected of the cluster. + +## The Brupop Custom Resource + +A resource is an endpoint in the Kubernetes API that stores a collection of +API objects of a certain kind; A custom resource is an extension of the +Kubernetes API that is not necessarily available in a default Kubernetes +installation. Kubernetes provides users with a mechanism for defining custom +objects to be managed by the API. + +In Brupop, update status of each node running Bottlerocket is tracked in the +“shadow” custom resource, we call it “BottlerocketShadow” or “brs” for short. + +## Bottlerocket shadow + +The BottlerocketShadow maintains status information for each node. + +The spec contains information about the desired state for the node. In +particular, the spec contains information about the desired Bottlerocket +version and desired state in the Brupop state machine. + +The status contains information about the desired latest version for the node, +as well as the current state machine state. + +```rust +pub enum BottlerocketShadowState + /// Nodes in this state are waiting for new updates to become available. + /// This is both the starting, terminal and recovery state in the update process. + Idle, + /// Nodes in this state have staged a new update image, have installed + /// the new image, and have updated the partition table + /// to mark it as the new active image. + StagedAndPerformedUpdate, + /// Nodes in this state have used the kubernetes cordon and drain APIs to remove + /// running pods, have un-cordoned the node to allow work to be scheduled, and + /// have rebooted after performing an update. + RebootedIntoUpdate, + /// Nodes in this state are monitoring to ensure that the node seems healthy + /// before marking the update as complete. + MonitoringUpdate, + /// Nodes in this state have crashed due to Bottlerocket Update API call failure. + ErrorReset, +} +``` + +### The controller component + +The controller component is a control loop responsible for managing the state of +the entire cluster. The controller maintains the logic of state transfer based +on our state machine. For a BottlerocketShadow, the controller uses the state +in status to determine state, and then set the state in spec to the next state +to drive updates. The defualt behavior is to update one node at a time. Users +may set the MAX_CONCURRENT_UPDATE environment variable in the controller’s +container to allow for multiple containers to update at the same time. + +### The agent component + +The agent component is another control loop responsible for managing the state +of an individual node. Agents gather system information using Bottlerocket’s +local API . It communicates with the API by mounting a unix socket into the +agent container. The agent updates the BottlerocketShadow status associated +with its node to include this gathered information, as well as the current +update state machine state. The agent then uses the BottlerocketShadow’s +spec to determine if it needs to make state transitions in the state +machine, and executes on them. + +When an update occurs, the agent cordons the node to prevent new pods from +being scheduled to it, and then drains it of any running pods ). After the +update-reboot, the node is uncordoned. + +All operations which write data to the Kubernetes API are routed through the +apiserver. This includes updates to the BottlerocketShadow status and +commands to drain and cordon the node, or uncordon it later. + +### The API server component + +The apiserver component performs all write requests to the Kubernetes API on +behalf of the Brupop agent. This is because Kubernetes uses role-based access +controls (RBAC). If RBAC were used to allow the agents direct access to modify +their own Shadow objects, agents would have the ability to write to the status +of all Shadow objects. To solve this issue, we choose to channel +bottlerocketshadow writes through a preveledged web API. The apiserver checks +request headers for a special token which is mounted into the agent pods by +Kubernetes, then uses the Kubernetes TokenReview APIs to assert that the +requesting node is certainly associated with the target BottlerocketShadow. +Further, we utilize certificates generated by cert-manager to ensure calls +to the api are generated from trusted sources. + +### Metrics endpoint + +Brupop exposes prometheus style service metrics via web servers hosted on +the Brupop API Server and Controller components. These are exposed via HTTP +at the standard /metrics endpoint. Currently, the endpoints surface request +metrics from the APIServer, a metric representing the current Bottlerocket +version, and a metric representing the current state of each node managed by +Brupop. Annotations are available on the default yaml manifest which enable +metrics to be automatically scrapped into available prometheus like metric +stores on the cluster. + +##### Monitoring cluster history and metrics with Prometheus + +[A sample configuration](https://github.com/bottlerocket-os/bottlerocket-update-operator/blob/develop/yamlgen/telemetry/prometheus-resources.yaml) +is provided which demonstrates a Prometheus deployment into a cluster that is +configured to gather metrics data from the update operator. Once Prometheus +is running in the cluster, you can use the Prometheus UI to visualize the +cluster's history. + +Using port-forward via kubectl: + +``` +kubectl port-forward pods/PROMETHEUS-POD-NAME 9090:9090 +``` + +You can then point a web browser to `localhost:9090` to see visualize the +update operator metrics and see updates in real time. + +# Contact us + +Please feel free to contact us [via GitHub](https://github.com/bottlerocket-os/bottlerocket-update-operator) +if you have any questions or feature requests! We are always happy to hear from +our community. You can also [join the Bottlerocket community meeting](https://www.meetup.com/bottlerocket-community/) +if you’d like to hear from us directly. + diff --git a/yamlgen/deploy/bottlerocket-update-operator.yaml b/yamlgen/deploy/bottlerocket-update-operator.yaml index b25ea0ab..d00e7735 100644 --- a/yamlgen/deploy/bottlerocket-update-operator.yaml +++ b/yamlgen/deploy/bottlerocket-update-operator.yaml @@ -384,7 +384,7 @@ spec: env: - name: APISERVER_INTERNAL_PORT value: "8443" - image: "public.ecr.aws/bottlerocket/bottlerocket-update-operator:v0.2.2" + image: "public.ecr.aws/bottlerocket/bottlerocket-update-operator:v1.0.0" livenessProbe: httpGet: path: /ping @@ -538,7 +538,7 @@ spec: value: "0" - name: APISERVER_SERVICE_PORT value: "443" - image: "public.ecr.aws/bottlerocket/bottlerocket-update-operator:v0.2.2" + image: "public.ecr.aws/bottlerocket/bottlerocket-update-operator:v1.0.0" name: brupop resources: limits: @@ -723,7 +723,7 @@ spec: value: "0:0:0" - name: UPDATE_WINDOW_STOP value: "0:0:0" - image: "public.ecr.aws/bottlerocket/bottlerocket-update-operator:v0.2.2" + image: "public.ecr.aws/bottlerocket/bottlerocket-update-operator:v1.0.0" name: brupop priorityClassName: brupop-controller-high-priority serviceAccountName: brupop-controller-service-account