[db backups] remove mysqldump, use rclone for s3

The primary change in this commit is to replace use of the "aws-cli"
tool with "rclone", as the AWS tools are not compatible with ceph rgw
based s3 implementations.

Additionally, this removes support for the mysqldump based legacy
backups, and refactors the backup script for readability and improved
logging.

Behavior changes:
Rather than only the latest backup file, rclone will copy any new files
that are not present in the object store, but will NOT overwrite
existing ones.

roles/chameleon_mariadb/defaults/main.yml

@@ -5,13 +5,17 @@ deployment_dir: "{{ deployment_dir }}"
 deployment_site_config_dir: "{{ site_config_dir }}"
 mariadb_backup_timer_suffix: "{{ site_config_dir | basename }}"
 mariadb_backup_cron_script: /usr/local/sbin/mariadb-backup
-mariadb_backup_max_packet: 1M
 mariadb_backup_file_age: 29
-mariadb_backup_s3_conf_dir: /etc/awsconf
-mariadb_backup_container_name: backup
+mariadb_backup_s3_endpoint:
+mariadb_backup_s3_key_id:
+mariadb_backup_s3_key:
+# if mariadb_backup_container_name is defined, use that
+mariadb_backup_bucket_name: "{{ mariadb_backup_container_name | default() }}"
 
 # these parameters are copied from kolla-ansible/ansible/roles/common/defaults/main.yml
 # because this playbook is executed outside the context of that role
 kolla_toolbox_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/kolla-toolbox"
 kolla_toolbox_tag: "{{ openstack_tag }}"
 kolla_toolbox_image_full: "{{ kolla_toolbox_image }}:{{ kolla_toolbox_tag }}"
+
+rclone_image: "rclone/rclone:1.67"

roles/chameleon_mariadb/tasks/config.yml

@@ -1,34 +1,11 @@
 ---
-- name: Create backup user
-  when: enable_mariabackup|bool == false
-  kolla_toolbox:
-    container_engine: "{{ kolla_container_engine }}"
-    module_name: mysql_user
-    module_args:
-      login_host: "{{ database_address }}"
-      login_user: "root"
-      login_password: "{{ database_password }}"
-      name: "{{ backup_database_user }}"
-      host: "%"
-      password: "{{ backup_database_password }}"
-      priv: "*.*:SELECT,RELOAD,LOCK TABLES"
-  delegate_to: "{{ groups['control'][0] }}"
-  run_once: True
-
 - name: Configure mariadb backup script file
-  become: yes
+  become: True
   template:
     src: mariadb.j2
     mode: 0700
     dest: "{{ mariadb_backup_cron_script }}"
 
-- name: Create directory to mount to aws-cli docker container for copying backup files to container
-  become: yes
-  when: mariadb_backup_s3_key_id is defined
-  file:
-    path: /etc/awsconf
-    state: directory
-
 - name: Add mariadb backup job
   include_role:
     name: chameleon.periodic_task

roles/chameleon_mariadb/templates/mariadb.j2

@@ -1,81 +1,45 @@
 #!/bin/bash
 #
 # MySQL Backup Script
-#
-
-ROTATE="{{ mariadb_backup_file_age }}"
-DIR="{{ backup_location }}"
-
-set -o pipefail
-
-{% if enable_mariabackup|bool == false %}
-#  Dumps mysql databases to a file
-##### START CONFIG ###################################################
-
-USER="{{ backup_database_user }}"
-PASS="{{ backup_database_password }}"
-MAX_ALLOWED_PACKET="{{ mariadb_backup_max_packet }}"
-
-# Create temporary mysql cnf file.
-TMPFILE=`mktemp /tmp/backup.XXXXXX` || exit 1
-cat >"$TMPFILE" <<EOF
-[client]
-password=$PASS
-user=$USER
-max_allowed_packet=$MAX_ALLOWED_PACKET
-EOF
-
-PREFIX=mysql_backup_
 
-ADDITIONAL_OPTIONS="--ignore-table=mysql.event"
+set -euo pipefail
 
-##### STOP CONFIG ####################################################
-backup_file_name=`date +%Y%m%d-%H%M%S`.sql.bz2
-backup_file=${DIR}/${PREFIX}${backup_file_name}
-
-cleanup() {
-  rm -f "$TMPFILE"
-  # Rotate backups
-  find "${DIR}/" -maxdepth 1 -type f -name "${PREFIX}*.sql*" -mtime +${ROTATE} -print0 | xargs -0 -r rm -f
-}
-
-mkdir -p $DIR
-mysqldump --defaults-extra-file="$TMPFILE" --opt --flush-logs --single-transaction \
- ${ADDITIONAL_OPTIONS} \
- --all-databases | bzcat -zc > "$backup_file"
-{% endif %}
-
-{% if enable_mariabackup|bool == true %}
 # mariadb backup is handled by kolla. just do cleanups here.
+ROTATE="{{ mariadb_backup_file_age }}"
+BACKUP_VOLUME="{{ backup_location }}"
 
 PREFIX=mysqlbackup-
-backup_file_name=$(docker run --rm -v ${DIR}:/backups "{{ kolla_toolbox_image_full }}" /bin/bash -c "ls -Art /backups/${PREFIX}* | tail -n 1")
-backup_file_name=$(basename $backup_file_name)
 
 cleanup() {
-  # Rotate backups  
-  docker run -u 0 --rm -v ${DIR}:/backup "{{ kolla_toolbox_image_full }}" \
+  # on exit, rotate backups keeping the last $ROTATE days
+  docker run -u 0 --rm -v ${BACKUP_VOLUME}:/backup "{{ kolla_toolbox_image_full }}" \
   /bin/bash -c \
-  "find /backup -maxdepth 1 -type f -name "${PREFIX}*" -mtime +${ROTATE} -print0 | xargs -0 -r rm -f"
+  "find /backup -maxdepth 1 -type f -name "${PREFIX}*" -mtime +${ROTATE} -print0 | xargs -0 -r rm -v -f"
 }
 
-{% endif %}
 
 trap cleanup EXIT
 
-{% if mariadb_backup_s3_endpoint is defined and mariadb_backup_s3_key_id is defined and mariadb_backup_s3_key is defined %}
-# Additionally copy backup to S3 bucket
-_aws() {
+backup_to_s3() {
+  # note: network=host is required when running on kolla control-plane hosts
+  # see "Docker iptables manipulation and bridge networking are now disabled by default"
+  # as per kolla-ansible wallaby upgrade notes
+
+  # see https://rclone.org/docs/#config-file for env_var syntax
   docker run --rm \
-    -e AWS_ACCESS_KEY_ID="{{ mariadb_backup_s3_key_id }}" \
-    -e AWS_SECRET_ACCESS_KEY="{{ mariadb_backup_s3_key }}" \
-    -v "{{ mariadb_backup_s3_conf_dir }}":/root/.aws \
-    -v ${DIR}:/backups \
-    amazon/aws-cli \
-    --endpoint="{{ mariadb_backup_s3_endpoint }}" \
-    --region=us-east-1 \
-    "$@"
+    --network host \
+    -e  "RCLONE_CONFIG_S3_TYPE=s3" \
+    -e  "RCLONE_CONFIG_S3_PROVIDER=Ceph" \
+    -e  "RCLONE_CONFIG_S3_ENDPOINT={{ mariadb_backup_s3_endpoint }}" \
+    -e  "RCLONE_CONFIG_S3_ACCESS_KEY_ID={{ mariadb_backup_s3_key_id }}" \
+    -e  "RCLONE_CONFIG_S3_SECRET_ACCESS_KEY={{ mariadb_backup_s3_key }}" \
+    -v "${BACKUP_VOLUME}:/backups:ro" \
+    "{{ rclone_image }}" \
+    "${@}"
 }
-_aws s3 mb s3://"{{ mariadb_backup_container_name }}"
-_aws s3 cp /backups/"$backup_file_name" s3://"{{ mariadb_backup_container_name }}"
-{% endif %}
+
+backup_to_s3 -v \
+	copy \
+	--ignore-existing \
+	"/backups/" \
+	"s3:/{{ mariadb_backup_bucket_name }}"