Adding working test candidate

kbase · Apr 9, 2024 · ef6b6c4 · ef6b6c4
1 parent 41fc18c
commit ef6b6c4
Showing 1 changed file with 48 additions and 63 deletions.
diff --git a/app/zip2cloud b/app/zip2cloud
@@ -13,24 +13,26 @@ ZIP_BASE=${ZIP_BASE:-backup_full}
 ZIP_DIR=${ZIP_DIR:-/zip}
 ZIP_RETENTION=${ZIP_RETENTION:-4}
 
+### Cleanup
+
 [ -r /run/secrets/encryption_key ] || { echo "Encryption key not readable in /run/secrets/encryption_key" ; exit 1; }
 [ -r /run/secrets/gcp_backup_creds ] || { echo "Google cloud service credentials not found in /run/secrets/gcp_back_creds" ; exit 1; }
 [ -z "${BUCKET}" ] && { echo "S3 bucketname not set in BUCKET environment variable" ; exit 1; }
 [ -z "${BUCKETPATH}" ] && { echo "Path within S3 bucket not set in BUCKETPATH environment variable" ; exit 1; }
 [ -z "${DELETE_DUMP}" ] || echo "DELETE_DUMP set, will delete files/directories under /dump/ when done compressing"
 
-# Delete any files older than 30 days in the zip directory
-#echo "Deleting database archives older than 30 days"
-#/usr/bin/find ${ZIP_DIR} -mtime +30 -type f -name "${ZIP_BASE}*" -print -exec rm {} \;
 
-# Delete all old zip files, except the last N+1, as defined by $ZIP_RETENTION
+# Delete all old zip files, except the last N, as defined by $ZIP_RETENTION
 rm -rf ${ZIP_DIR}/tmp_md5
 ls -t ${ZIP_DIR}/*.7z 2>/dev/null | tail -n +${ZIP_RETENTION} | xargs -r rm -f
 ls -t ${ZIP_DIR}/*.md5 2>/dev/null | tail -n +${ZIP_RETENTION}  | xargs -r rm -f
 
-# Delete all old backup dumps, except the last N+1, as defined by $DUMP_RETENTION
+# Delete all old backup dumps, except the last N, as defined by $DUMP_RETENTION
 find ${DUMP_BASE} -type d -name "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]" -print0 | xargs -0 ls -td | tail -n +${DUMP_RETENTION} | xargs -I {} rm -rf {}
 
+### End Cleanup
+
+
 # Get list of remote backups
 remote_files=$(rclone ls remote:${BUCKET}/${BUCKETPATH} | grep 7z | awk '{print $2}' | rev | cut -d. -f2- | rev)
 # Pull remote md5 sums for each remote backup into `tmp_md5` directory
@@ -39,59 +41,63 @@ for file in $remote_files; do
     rclone md5sum remote:${BUCKET}/${BUCKETPATH}/$file.7z | awk '{print $1}' > ${ZIP_DIR}/tmp_md5/$file.md5
 done
 
-# Create empty list of files to upload
-uploads=""
-
-echo "Zipping ${DUMP_BASE}/${DUMP_DIR} to ${ZIP_DIR}/${ZIP_NAME}"
 
-# Get all directories in DUMP_BASE
+# Get all exports from DUMP_BASE
 for DUMP_DIR in $(ls -d ${DUMP_BASE}/*/); do
     # Remove trailing slash and get the base name of the directory
     DIR_NAME=$(basename ${DUMP_DIR%/})
     ZIP_NAME=${ZIP_DIR}/${ZIP_BASE}_${DIR_NAME}
     echo $DIR_NAME
 
-    # Check if the corresponding md5 file exists
-    if [ -f "${ZIP_DIR}/tmp_md5/${ZIP_BASE}_${DIR_NAME}.md5" ]; then
-        echo "MD5 file exists for ${DIR_NAME}, skipping"
-        continue
+    # Check if the corresponding md5 file exists, if not, zip it
+    if [ ! -f "${ZIP_DIR}/tmp_md5/${ZIP_BASE}_${DIR_NAME}.md5" ]; then
+      echo "No remote exists for ${DIR_NAME}, zipping"
+      /usr/bin/7z a -p${SECRET} ${ZIP_NAME}  -mx=${COMPRESSION_LEVEL} -mhe -t7z ${DUMP_DIR} || { echo "Could not zip ${DUMP_DIR} into ${ZIP_NAME}" ; exit 1; }
     fi
-
-    echo "Zipping ${DUMP_DIR} to ${ZIP_NAME}".7z
-    /usr/bin/7z a -p${SECRET} ${ZIP_NAME}  -mx=${COMPRESSION_LEVEL} -mhe -t7z ${DUMP_DIR} || { echo "Could not zip ${DUMP_DIR} into ${ZIP_NAME}" ; exit 1; }
 done
 
-# Create md5 sums for local backups, if they don't exist
-cd ${ZIP_DIR}
+# Compare checksums of local 7z files against all remotes' md5's. Add to upload list if not found
+uploads=""
+cd ${ZIP_DIR} || exit
 for file in ${ZIP_DIR}/*.7z; do
     # Get the base name of the file without extension
     base_name=$(basename "$file" .7z)
-    echo $base_name
-    # If a local .md5 file does not exist, create it
-    if [ ! -f "${ZIP_DIR}/${base_name}.md5" ]; then
-        echo "Local md5 file does not exist for $file, generating, and adding $file to uploads list"
+    local_md5=$(md5sum "$file" | awk '{print $1}')
+    echo $local_md5 > "${ZIP_DIR}/${base_name}.md5"
+    # Now compare this file with the remote md5s
+    match_found=0
+    for remote_md5_file in ${ZIP_DIR}/tmp_md5/*.md5; do
+        remote_md5=$(cat "$remote_md5_file")
+        if [ "$local_md5" = "$remote_md5" ]; then
+            match_found=1
+            break
+        fi
+    done
+    if [ $match_found -eq 0 ]; then
+        echo "Adding $file to uploads list"
         uploads="$uploads $file"
-        local_md5=$(md5sum "$file" | awk '{print $1}')
-        echo $local_md5 > "${ZIP_DIR}/${base_name}.md5"
     fi
 done
 
-# Verify & update list of files to upload
-cd ${ZIP_DIR}/
-for file in ${ZIP_DIR}/*.7z; do
+echo "Current uploads candidates are: $uploads"
+
+## Verify & update list of files to upload
+final_uploads=""
+cd ${ZIP_DIR} || exit
+for file in ${uploads}; do
     # Get the base name of the file without extension
     base_name=$(basename "$file" .7z)
-    # Check if the remote md5 file exists
-    if [ ! -f "${ZIP_DIR}/tmp_md5/${base_name}.md5" ]; then
-        # If the remote md5 file does not exist, add the file to the uploads list
-        echo "Remote does not exist for $file, adding $file to uploads list"
-        uploads="$uploads $file"
-    else
+#    # Check if the remote md5 file exists
+#    if [ ! -f "${ZIP_DIR}/tmp_md5/${base_name}.md5" ]; then
+#        # If the remote md5 file does not exist, add the file to the uploads list
+#        echo "Remote does not exist for $file, adding $file to uploads list"
+#        final_uploads="$final_uploads $file"
+#    else
       # Compare local and remote md5
        remote_md5=$(cat "${ZIP_DIR}/tmp_md5/${base_name}.md5")
        local_md5=$(cat "${ZIP_DIR}/${base_name}.md5")
        if [ "$local_md5" != "$remote_md5" ]; then
-           echo "MD5 mismatch for file $file"
+           echo "MD5 mismatch for file $file. Incrementing filename and adding to uploads list."
            # Extract the last character of the base name
            last_char=${base_name: -1}
            # Check if the last character is a letter
@@ -109,37 +115,16 @@ for file in ${ZIP_DIR}/*.7z; do
            # Rename the file
            mv "$file" "${ZIP_DIR}/${new_base_name}.7z"
            # Add the renamed file to the uploads list
-           uploads="$uploads ${ZIP_DIR}/${new_base_name}.7z"
-       fi
-#        # Compare local and remote md5
-#        remote_md5=$(cat "${ZIP_DIR}/tmp_md5/${base_name}.md5")
-#        local_md5=$(cat "${ZIP_DIR}/${base_name}.md5")
-#        if [ "$local_md5" != "$remote_md5" ]; then
-#            echo "MD5 mismatch for file $file"
-#            # Extract the last character of the base name
-#            last_char=${base_name: -1}
-#            # Check if the last character is a letter
-#            if [[ $last_char =~ [a-z] ]]; then
-#                # If it's a letter, increment it
-#                next_char=$(echo "$last_char" | tr "a-y" "b-z")
-#                new_base_name=${base_name%?}$next_char
-#            else
-#                # If it's not a letter, append 'a'
-#                new_base_name=${base_name}a
-#            fi
-#            # Rename the file
-#            mv "$file" "${ZIP_DIR}/${new_base_name}.7z"
-#            # Add the renamed file to the uploads list
-#            uploads="$uploads ${ZIP_DIR}/${new_base_name}.7z"
-#        fi
-    fi
-    echo "Uploads: $uploads"
+           final_uploads="$final_uploads ${ZIP_DIR}/${new_base_name}.7z"
+        fi
 done
 
+echo "Final uploads: $final_uploads"
+
 
 # Before running rclone
 #for file in "${uploads[@]}"; do
-for file in ${uploads}; do
+for file in ${final_uploads}; do
     ls $file
     if [ ! -f "$file" ]; then
         echo "File does not exist: $file"
@@ -149,8 +134,8 @@ done
 
 
 ## Sync All Resulting Files (in list!)
-cd ${ZIP_DIR}
-for file in ${uploads}; do
+cd ${ZIP_DIR} || exit
+for file in ${final_uploads}; do
     echo "RClone-ing ${file} to GCP ${REMOTE}"
     /usr/bin/rclone sync -v "$file" ${REMOTE}/
 done