Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DEVOPS-1770: Adding spacing for log clarity #33

Merged
merged 1 commit into from
May 9, 2024
Merged

DEVOPS-1770: Adding spacing for log clarity #33

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions app/zip2cloud
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ ZIP_RETENTION=${ZIP_RETENTION:-4}

### Cleanup

[ -r /run/secrets/encryption_key ] || { echo "Encryption key not readable in /run/secrets/encryption_key" ; exit 1; }
[ -r /run/secrets/gcp_backup_creds ] || { echo "Google cloud service credentials not found in /run/secrets/gcp_back_creds" ; exit 1; }
[ -z "${BUCKET}" ] && { echo "S3 bucketname not set in BUCKET environment variable" ; exit 1; }
[ -z "${BUCKETPATH}" ] && { echo "Path within S3 bucket not set in BUCKETPATH environment variable" ; exit 1; }
[ -z "${DELETE_DUMP}" ] || echo "DELETE_DUMP set, will delete files/directories under /dump/ when done compressing"
[ -r /run/secrets/encryption_key ] || { echo " Encryption key not readable in /run/secrets/encryption_key" ; exit 1; }
[ -r /run/secrets/gcp_backup_creds ] || { echo " Google cloud service credentials not found in /run/secrets/gcp_back_creds" ; exit 1; }
[ -z "${BUCKET}" ] && { echo " S3 bucketname not set in BUCKET environment variable" ; exit 1; }
[ -z "${BUCKETPATH}" ] && { echo " Path within S3 bucket not set in BUCKETPATH environment variable" ; exit 1; }
[ -z "${DELETE_DUMP}" ] || echo " DELETE_DUMP set, will delete files/directories under /dump/ when done compressing"


# Delete all old zip files, except the last N, as defined by $ZIP_RETENTION
Expand Down Expand Up @@ -50,9 +50,9 @@ done
# Get all exports from DUMP_BASE
for DUMP_DIR in $(ls -d ${DUMP_BASE}/*/); do
# Check if the dump is complete
echo "Checking export for ${DUMP_DIR}"
echo " Checking export for ${DUMP_DIR}"
if [ ! -f "${DUMP_DIR}/dump_complete.txt" ]; then
echo "dump_complete.txt not found in ${DUMP_DIR}, skipping"
echo " dump_complete.txt not found in ${DUMP_DIR}, skipping"
continue
fi
# Remove trailing slash and get the base name of the directory
Expand All @@ -61,18 +61,18 @@ for DUMP_DIR in $(ls -d ${DUMP_BASE}/*/); do
echo $DIR_NAME
# Check if the corresponding md5 file exists, if not, zip it
if [ ! -f "${ZIP_DIR}/tmp_md5/${ZIP_BASE}_${DIR_NAME}.md5" ]; then
echo "No remote exists for ${DIR_NAME}, zipping"
/usr/bin/7z a -p${SECRET} ${ZIP_NAME} -mx=${COMPRESSION_LEVEL} -mhe -t7z ${DUMP_DIR} || { echo "Could not zip ${DUMP_DIR} into ${ZIP_NAME}" ; exit 1; }
echo " No remote exists for ${DIR_NAME}, zipping"
/usr/bin/7z a -p${SECRET} ${ZIP_NAME} -mx=${COMPRESSION_LEVEL} -mhe -t7z ${DUMP_DIR} || { echo " Could not zip ${DUMP_DIR} into ${ZIP_NAME}" ; exit 1; }
else
echo "Remote exists for ${DIR_NAME}, skipping"
echo " Remote exists for ${DIR_NAME}, skipping"
fi
done

# Compare checksums of local 7z files against all remotes' md5's. Add to upload list if not found
uploads=""
cd ${ZIP_DIR} || exit
for file in ${ZIP_DIR}/*.7z; do
echo "Comparing $file with all remote md5s"
echo " Comparing $file with all remote md5s"
# Get the base name of the file without extension
base_name=$(basename "$file" .7z)
local_md5=$(md5sum "$file" | awk '{print $1}')
Expand All @@ -81,20 +81,20 @@ for file in ${ZIP_DIR}/*.7z; do
match_found=0
for remote_md5_file in ${ZIP_DIR}/tmp_md5/*.md5; do
remote_md5=$(cat "$remote_md5_file")
echo "Comparing $file with $remote_md5_file"
echo " Comparing $file with $remote_md5_file"
if [ "$local_md5" = "$remote_md5" ]; then
match_found=1
echo "$file checksum matches $remote_md5_file, skipping"
echo " $file checksum matches $remote_md5_file, skipping"
break
fi
done
if [ $match_found -eq 0 ]; then
echo "Adding $file to uploads list"
echo " Adding $file to uploads list"
uploads="$uploads $file"
fi
done

echo "Current uploads candidates are: $uploads"
echo " Current uploads candidates are: $uploads"

### End Checksumming

Expand All @@ -110,7 +110,7 @@ for file in ${uploads}; do
remote_md5=$(cat "${ZIP_DIR}/tmp_md5/${base_name}.md5")
local_md5=$(cat "${ZIP_DIR}/${base_name}.md5")
if [ "$local_md5" != "$remote_md5" ]; then
echo "MD5 mismatch for file $file. Incrementing filename and adding to uploads list."
echo " MD5 mismatch for file $file. Incrementing filename and adding to uploads list."
# Extract the last character of the base name
last_char=${base_name: -1}
# Check if the last character is a letter
Expand Down Expand Up @@ -142,7 +142,7 @@ for file in ${uploads}; do
final_uploads="$final_uploads ${ZIP_DIR}/${base_name}.7z"
fi
done
echo "Final uploads: $final_uploads"
echo " Final uploads: $final_uploads"

### End Create Upload List

Expand All @@ -152,7 +152,7 @@ echo "Final uploads: $final_uploads"
for file in ${final_uploads}; do
ls $file
if [ ! -f "$file" ]; then
echo "File does not exist: $file"
echo " File does not exist: $file"
fi
done

Expand All @@ -161,7 +161,7 @@ done
## Sync All Resulting Files (in list!)
if [ "$ENABLE_UPLOAD" = true ]; then
for file in ${final_uploads}; do
echo "RClone-ing ${file} to GCP ${REMOTE}"
echo " RClone-ing ${file} to GCP ${REMOTE}"
/usr/bin/rclone sync -v "$file" ${REMOTE}/
done
fi
Expand Down
Loading