Merge pull request #538 from stackhpc/hotfix-containers-playbook

Add playbooks to allow hotfixing containers

doc/source/operations/hotfix-playbook.rst

@@ -0,0 +1,70 @@
+===============
+Hotfix Playbook
+===============
+
+Using the Container Hotfix Playbook
+===================================
+
+The StackHPC Kayobe configuration contains a playbook called
+``hotfix-containers.yml`` which can be used to execute commands on, and copy
+files into, a given set of containers.
+
+This playbook will first copy across any hotfix files, and then run the
+hotfix command. If either of these are not specified, the corresponding step
+will be skipped.
+
+This playbook is designed for use in high-severity hotfixes ONLY and should not
+be used for regular operations.
+
+The playbook can be invoked with:
+
+.. code-block:: console
+
+  kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/hotfix-containers.yml
+
+Playbook variables:
+-------------------
+
+* ``container_hotfix_command``: A command to run on each of the target
+  containers. Default is an empty string.
+
+* ``container_hotfix_files``: A list of files to copy into each target
+  container. Consists of a list of dicts with keys ``src`` and ``dest``
+  (required), and ``mode`` (optional - default 400). Default is an empty list.
+
+* ``container_hotfix_container_regex``: Regex to match container names against.
+  Must match the entire name e.g. "nova" or "nova*" will result in only
+  matching a single container called "nova". To properly match every container
+  starting with "nova", the regex must be "nova.*" Default is an empty string.
+
+* ``container_hotfix_restart_containers``: Whether to restart containers after
+  applying the hotfix. Default is False.
+
+* ``container_hotfix_become``: Create files and exec as root in the target
+  containers. Default is False.
+
+
+It is strongly recommended that you write your container_hotfix_* variables
+to a file, then add them as an extra var. e.g:
+
+.. code-block:: console
+
+  kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/hotfix-containers.yml -e "@~/vars.yml"
+
+
+Example Variables file
+----------------------
+
+.. code-block:: yaml
+
+    ---
+    container_hotfix_command: "/tmp/quick-fix.sh"
+    container_hotfix_files:
+      - src: "~/quick-fix.sh"
+        dest: "/tmp/quick-fix.sh"
+        mode: "700"
+      - src: "/home/stackhpc/new_nova_conf.conf"
+        dest: "/etc/nova/nova.conf"
+    container_hotfix_container_regex: "nova.*"
+    container_hotfix_restart_containers: True
+    container_hotfix_become: True

doc/source/operations/index.rst

@@ -9,3 +9,4 @@ This guide is for operators of the StackHPC Kayobe configuration project.
 
    rabbitmq
    octavia
+   hotfix-playbook

etc/kayobe/ansible/hotfix-containers.yml

@@ -0,0 +1,70 @@
+---
+# NOTE: This playbook is designed for use in high-severity hotfixes ONLY.
+# If you're considering using this for regular operations, please consider
+# developing a more suitable solution instead.
+#
+# See https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-yoga/operations/hotfix-containers.html
+# for more information.
+
+- name: Hotfix containers
+  hosts: overcloud
+  tags:
+    - hotfix-containers
+  vars:
+    container_hotfix_command: ""
+    container_hotfix_files: []
+    container_hotfix_container_regex: ""
+    container_hotfix_restart_containers: False
+    container_hotfix_become: False
+  tasks:
+    - name: Ensure inputs are valid
+      fail:
+        msg: "Invalid input. Container list cannot be empty. Either container_hotfix_command or container_hotfix_files must be populated."
+      when:
+        - container_hotfix_container_regex == "" or
+          container_hotfix_command == "" and container_hotfix_files == []
+
+    - name: Get list of containers to hotfix
+      command: '{{ kolla_container_engine | default("docker")}} ps --format {% raw %}"{{.Names}}"{% endraw %}'
+      register: host_containers
+
+    - name: Set fact for containers list
+      set_fact:
+        containers_list: host_containers.stdout
+
+    - name: Fail if no containers match given regex
+      vars:
+        hotfix_containers: "{{ containers_list | split('\n') | regex_search(container_hotfix_container_regex) }}"
+      fail:
+        msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers | split('\n') }}"
+      when: hotfix_containers == ""
+
+    - name: Ensure hotfix-files directory exists on the remote host
+      ansible.builtin.file:
+        path: /tmp/hotfix-files
+        state: directory
+
+    - name: Ensure container hotfix file(s) exist on host
+      ansible.builtin.copy:
+        src: "{{ item.src }}"
+        dest: "/tmp/hotfix-files/{{ index }}"
+      loop: "{{ container_hotfix_files }}"
+      loop_control:
+        index_var: index
+      when: container_hotfix_files != []
+
+    - name: Apply hotfix
+      include_tasks: run-container-hotfix.yml
+      loop: "{{ containers_list | regex_findall(container_hotfix_container_regex, multiline=True) | list | unique }}"
+      loop_control:
+        loop_var: hotfix_container
+
+    - name: Cleanup temporary files
+      ansible.builtin.file:
+        path: /tmp/hotfix-files
+        state: absent
+
+    - name: Restart containers if requested
+      command: "{{ kolla_container_engine | default('docker')}} restart {{ item }}"
+      loop: "{{ containers_list | regex_findall(container_hotfix_container_regex, multiline=True) | list | unique }}"
+      when: container_hotfix_restart_containers

etc/kayobe/ansible/run-container-hotfix.yml

@@ -0,0 +1,22 @@
+---
+- block:
+    - name: Ensure any required directories exist in container(s)
+      command: "{{ kolla_container_engine | default('docker') }} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} mkdir -p {{ item.dest | dirname }}"
+      loop: "{{ container_hotfix_files }}"
+
+    - name: Copy file into container(s)
+      command: "{{ kolla_container_engine | default('docker') }} cp /tmp/hotfix-files/{{ index }} {{ hotfix_container }}:{{ item.dest }}"
+      loop: "{{ container_hotfix_files }}"
+      loop_control:
+        index_var: index
+
+    - name: Set mode for copied files
+      command: "{{ kolla_container_engine | default('docker') }} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} chmod {{ item.mode | default('400') }} {{ item.dest }}"
+      loop: "{{ container_hotfix_files }}"
+      loop_control:
+        index_var: index
+
+  when: container_hotfix_files != []
+
+- name: Run container_hotfix_command
+  command: "{{ kolla_container_engine | default('docker')}} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} {{ container_hotfix_command }}"

releasenotes/notes/add-hotfix-containers-playbook-5a6d3d48067cf0b6.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    The playbook ``hotfix-containers.yml`` has been added. This allows
+    arbitrary files to be copied into, and/or arbitrary commands to be executed
+    within, overcloud containers.