From 923f3cf94db3a8fb8f7846e644b769a32fa0c20a Mon Sep 17 00:00:00 2001
From: Michael Sherman <shermanm@uchicago.edu>
Date: Fri, 2 Aug 2024 14:50:54 -0500
Subject: [PATCH] [db backups] remove mysqldump, use rclone for s3

The primary change in this commit is to replace use of the "aws-cli"
tool with "rclone", as the AWS tools are not compatible with ceph rgw
based s3 implementations.

Additionally, this removes support for the mysqldump based legacy
backups, and refactors the backup script for readability and improved
logging.

Behavior changes:
Rather than only the latest backup file, rclone will copy any new files
that are not present in the object store, but will NOT overwrite
existing ones.
---
 roles/chameleon_mariadb/defaults/main.yml    | 14 ++-
 roles/chameleon_mariadb/tasks/config.yml     | 25 +-----
 roles/chameleon_mariadb/templates/mariadb.j2 | 90 ++++++--------------
 3 files changed, 38 insertions(+), 91 deletions(-)

diff --git a/roles/chameleon_mariadb/defaults/main.yml b/roles/chameleon_mariadb/defaults/main.yml
index 07b2e748..7c7bbd4a 100644
--- a/roles/chameleon_mariadb/defaults/main.yml
+++ b/roles/chameleon_mariadb/defaults/main.yml
@@ -5,11 +5,17 @@ deployment_dir: "{{ deployment_dir }}"
 deployment_site_config_dir: "{{ site_config_dir }}"
 mariadb_backup_timer_suffix: "{{ site_config_dir | basename }}"
 mariadb_backup_cron_script: /usr/local/sbin/mariadb-backup
-mariadb_backup_max_packet: 1M
 mariadb_backup_file_age: 29
-mariadb_backup_s3_conf_dir: /etc/awsconf
-mariadb_backup_container_name: backup
+mariadb_backup_s3_endpoint:
+mariadb_backup_s3_key_id:
+mariadb_backup_s3_key:
+# if mariadb_backup_container_name is defined, use that
+mariadb_backup_bucket_name: "{{ mariadb_backup_container_name | default() }}"
 
+# these parameters are copied from kolla-ansible/ansible/roles/common/defaults/main.yml
+# because this playbook is executed outside the context of that role
 kolla_toolbox_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/kolla-toolbox"
-kolla_toolbox_tag: "{{ openstack_release }}"
+kolla_toolbox_tag: "{{ openstack_tag }}"
 kolla_toolbox_image_full: "{{ kolla_toolbox_image }}:{{ kolla_toolbox_tag }}"
+
+rclone_image: "rclone/rclone:1.67"
\ No newline at end of file
diff --git a/roles/chameleon_mariadb/tasks/config.yml b/roles/chameleon_mariadb/tasks/config.yml
index 53497d66..34038632 100644
--- a/roles/chameleon_mariadb/tasks/config.yml
+++ b/roles/chameleon_mariadb/tasks/config.yml
@@ -1,34 +1,11 @@
 ---
-- name: Create backup user
-  when: enable_mariabackup|bool == false
-  kolla_toolbox:
-    container_engine: "{{ kolla_container_engine }}"
-    module_name: mysql_user
-    module_args:
-      login_host: "{{ database_address }}"
-      login_user: "root"
-      login_password: "{{ database_password }}"
-      name: "{{ backup_database_user }}"
-      host: "%"
-      password: "{{ backup_database_password }}"
-      priv: "*.*:SELECT,RELOAD,LOCK TABLES"
-  delegate_to: "{{ groups['control'][0] }}"
-  run_once: True
-
 - name: Configure mariadb backup script file
-  become: yes
+  become: True
   template:
     src: mariadb.j2
     mode: 0700
     dest: "{{ mariadb_backup_cron_script }}"
 
-- name: Create directory to mount to aws-cli docker container for copying backup files to container
-  become: yes
-  when: mariadb_backup_s3_key_id is defined
-  file:
-    path: /etc/awsconf
-    state: directory
-
 - name: Add mariadb backup job
   include_role:
     name: chameleon.periodic_task
diff --git a/roles/chameleon_mariadb/templates/mariadb.j2 b/roles/chameleon_mariadb/templates/mariadb.j2
index 663282be..7f9288f3 100644
--- a/roles/chameleon_mariadb/templates/mariadb.j2
+++ b/roles/chameleon_mariadb/templates/mariadb.j2
@@ -1,81 +1,45 @@
 #!/bin/bash
 #
 # MySQL Backup Script
-#
-
-ROTATE="{{ mariadb_backup_file_age }}"
-DIR="{{ backup_location }}"
-
-set -o pipefail
-
-{% if enable_mariabackup|bool == false %}
-#  Dumps mysql databases to a file
-##### START CONFIG ###################################################
-
-USER="{{ backup_database_user }}"
-PASS="{{ backup_database_password }}"
-MAX_ALLOWED_PACKET="{{ mariadb_backup_max_packet }}"
-
-# Create temporary mysql cnf file.
-TMPFILE=`mktemp /tmp/backup.XXXXXX` || exit 1
-cat >"$TMPFILE" <<EOF
-[client]
-password=$PASS
-user=$USER
-max_allowed_packet=$MAX_ALLOWED_PACKET
-EOF
-
-PREFIX=mysql_backup_
 
-ADDITIONAL_OPTIONS="--ignore-table=mysql.event"
+set -euo pipefail
 
-##### STOP CONFIG ####################################################
-backup_file_name=`date +%Y%m%d-%H%M%S`.sql.bz2
-backup_file=${DIR}/${PREFIX}${backup_file_name}
-
-cleanup() {
-  rm -f "$TMPFILE"
-  # Rotate backups
-  find "${DIR}/" -maxdepth 1 -type f -name "${PREFIX}*.sql*" -mtime +${ROTATE} -print0 | xargs -0 -r rm -f
-}
-
-mkdir -p $DIR
-mysqldump --defaults-extra-file="$TMPFILE" --opt --flush-logs --single-transaction \
- ${ADDITIONAL_OPTIONS} \
- --all-databases | bzcat -zc > "$backup_file"
-{% endif %}
-
-{% if enable_mariabackup|bool == true %}
 # mariadb backup is handled by kolla. just do cleanups here.
+ROTATE="{{ mariadb_backup_file_age }}"
+BACKUP_VOLUME="{{ backup_location }}"
 
 PREFIX=mysqlbackup-
-backup_file_name=$(docker run --rm -v ${DIR}:/backups "{{ kolla_toolbox_image_full }}" /bin/bash -c "ls -Art /backups/${PREFIX}* | tail -n 1")
-backup_file_name=$(basename $backup_file_name)
 
 cleanup() {
-  # Rotate backups  
-  docker run -u 0 --rm -v ${DIR}:/backup "{{ kolla_toolbox_image_full }}" \
+  # on exit, rotate backups keeping the last $ROTATE days
+  docker run -u 0 --rm -v ${BACKUP_VOLUME}:/backup "{{ kolla_toolbox_image_full }}" \
   /bin/bash -c \
-  "find /backup -maxdepth 1 -type f -name "${PREFIX}*" -mtime +${ROTATE} -print0 | xargs -0 -r rm -f"
+  "find /backup -maxdepth 1 -type f -name "${PREFIX}*" -mtime +${ROTATE} -print0 | xargs -0 -r rm -v -f"
 }
 
-{% endif %}
 
 trap cleanup EXIT
 
-{% if mariadb_backup_s3_endpoint is defined and mariadb_backup_s3_key_id is defined and mariadb_backup_s3_key is defined %}
-# Additionally copy backup to S3 bucket
-_aws() {
+backup_to_s3() {
+  # note: network=host is required when running on kolla control-plane hosts
+  # see "Docker iptables manipulation and bridge networking are now disabled by default"
+  # as per kolla-ansible wallaby upgrade notes
+
+  # see https://rclone.org/docs/#config-file for env_var syntax
   docker run --rm \
-    -e AWS_ACCESS_KEY_ID="{{ mariadb_backup_s3_key_id }}" \
-    -e AWS_SECRET_ACCESS_KEY="{{ mariadb_backup_s3_key }}" \
-    -v "{{ mariadb_backup_s3_conf_dir }}":/root/.aws \
-    -v ${DIR}:/backups \
-    amazon/aws-cli \
-    --endpoint="{{ mariadb_backup_s3_endpoint }}" \
-    --region=us-east-1 \
-    "$@"
+    --network host \
+    -e  "RCLONE_CONFIG_S3_TYPE=s3" \
+    -e  "RCLONE_CONFIG_S3_PROVIDER=Ceph" \
+    -e  "RCLONE_CONFIG_S3_ENDPOINT={{ mariadb_backup_s3_endpoint }}" \
+    -e  "RCLONE_CONFIG_S3_ACCESS_KEY_ID={{ mariadb_backup_s3_key_id }}" \
+    -e  "RCLONE_CONFIG_S3_SECRET_ACCESS_KEY={{ mariadb_backup_s3_key }}" \
+    -v "${BACKUP_VOLUME}:/backups:ro" \
+    "{{ rclone_image }}" \
+    "${@}"
 }
-_aws s3 mb s3://"{{ mariadb_backup_container_name }}"
-_aws s3 cp /backups/"$backup_file_name" s3://"{{ mariadb_backup_container_name }}"
-{% endif %}
+
+backup_to_s3 -v \
+	copy \
+	--ignore-existing \
+	"/backups/" \
+	"s3:/{{ mariadb_backup_bucket_name }}"