-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
JS Fillman
committed
Apr 5, 2024
1 parent
dc8f23a
commit c1a8583
Showing
2 changed files
with
81 additions
and
181 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,27 +1,15 @@ | ||
| # Builder stage | ||
| FROM alpine:latest as builder | ||
| # Build stage | ||
| FROM golang:1.16 AS builder | ||
|
|
||
| RUN apk update && \ | ||
| apk add --no-cache curl p7zip rclone | ||
| WORKDIR /app | ||
|
|
||
| # Create config directory | ||
| RUN mkdir -p /root/.config/rclone/ | ||
| COPY . . | ||
|
|
||
| # Copy necessary files | ||
| COPY rclone.conf /root/.config/rclone/rclone.conf | ||
| COPY app/ /app/ | ||
| RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app . | ||
|
|
||
| # Final stage | ||
| FROM alpine:latest | ||
|
|
||
| RUN apk update && \ | ||
| apk add --no-cache curl p7zip | ||
| FROM scratch | ||
|
|
||
| # Copy necessary binaries and files from builder stage | ||
| COPY --from=builder /usr/bin/rclone /usr/bin/rclone | ||
| COPY --from=builder /root/.config/rclone/rclone.conf /root/.config/rclone/rclone.conf | ||
| COPY --from=builder /app/ /app/ | ||
|
|
||
| WORKDIR /app | ||
| COPY --from=builder /app/app /app | ||
|
|
||
| ENTRYPOINT ["/app/zip2cloud"] | ||
| ENTRYPOINT ["/app"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,161 +1,73 @@ | ||
| #!/bin/sh | ||
|
|
||
| ## Variables | ||
| COMPRESSION_LEVEL=${COMPRESSION_LEVEL:-0} # Set to 0 if the db dumps are already compressed | ||
| DELETE_DUMP=${DELETE_DUMP:-''} | ||
| DUMP_BASE=${DUMP_BASE:-/dump/full_backup} | ||
| DUMP_RETENTION=${DUMP_RETENTION:-3} | ||
| REMOTE=${REMOTE:-remote:${BUCKET}/${BUCKETPATH}} | ||
| SECRET=${SECRET:-`cat /run/secrets/encryption_key`} | ||
| SLACK_CHANNEL=${SLACK_CHANNEL:-''} | ||
| SLACK_WEBHOOK=${SLACK_WEBHOOK:-''} | ||
| ZIP_BASE=${ZIP_BASE:-backup_full} | ||
| ZIP_DIR=${ZIP_DIR:-/zip} | ||
| ZIP_RETENTION=${ZIP_RETENTION:-4} | ||
|
|
||
| [ -r /run/secrets/encryption_key ] || { echo "Encryption key not readable in /run/secrets/encryption_key" ; exit 1; } | ||
| [ -r /run/secrets/gcp_backup_creds ] || { echo "Google cloud service credentials not found in /run/secrets/gcp_back_creds" ; exit 1; } | ||
| [ -z "${BUCKET}" ] && { echo "S3 bucketname not set in BUCKET environment variable" ; exit 1; } | ||
| [ -z "${BUCKETPATH}" ] && { echo "Path within S3 bucket not set in BUCKETPATH environment variable" ; exit 1; } | ||
| [ -z "${DELETE_DUMP}" ] || echo "DELETE_DUMP set, will delete files/directories under /dump/ when done compressing" | ||
|
|
||
| # Delete any files older than 30 days in the zip directory | ||
| #echo "Deleting database archives older than 30 days" | ||
| #/usr/bin/find ${ZIP_DIR} -mtime +30 -type f -name "${ZIP_BASE}*" -print -exec rm {} \; | ||
|
|
||
| # Delete all old zip files, except the last N+1, as defined by $ZIP_RETENTION | ||
| ls -t ${ZIP_DIR}/*.7z 2>/dev/null | tail -n +$((${ZIP_RETENTION} - 1)) | xargs -r rm -f | ||
| ls -t ${ZIP_DIR}/*.md5 2>/dev/null | tail -n +$((${ZIP_RETENTION} - 1)) | xargs -r rm -f | ||
|
|
||
| # Delete all old backup dumps, except the last N+1, as defined by $DUMP_RETENTION | ||
| find ${DUMP_BASE} -type d -name "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]" -print0 | xargs -0 ls -td | tail -n +$((${DUMP_RETENTION} - 1)) | xargs -I {} rm -rf {} | ||
|
|
||
| # Get list of remote backups | ||
| remote_files=$(rclone ls remote:${BUCKET}/${BUCKETPATH} | grep 7z | awk '{print $2}' | rev | cut -d. -f2- | rev) | ||
| # Pull remote md5 sums for each remote backup into `tmp_md5` directory | ||
| mkdir -p ${ZIP_DIR}/tmp_md5 && cd $_ | ||
| for file in $remote_files; do | ||
| rclone md5sum remote:${BUCKET}/${BUCKETPATH}/$file.7z | awk '{print $1}' > ${ZIP_DIR}/tmp_md5/$file.md5 | ||
| done | ||
|
|
||
| # Create empty list of files to upload | ||
| uploads="" | ||
|
|
||
| echo "Zipping ${DUMP_BASE}/${DUMP_DIR} to ${ZIP_DIR}/${ZIP_NAME}" | ||
|
|
||
| # Get all directories in DUMP_BASE | ||
| for DUMP_DIR in $(ls -d ${DUMP_BASE}/*/); do | ||
| # Remove trailing slash and get the base name of the directory | ||
| DIR_NAME=$(basename ${DUMP_DIR%/}) | ||
| ZIP_NAME=${ZIP_DIR}/${ZIP_BASE}_${DIR_NAME} | ||
| echo $DIR_NAME | ||
|
|
||
| # Check if the corresponding md5 file exists | ||
| if [ -f "${ZIP_DIR}/tmp_md5/${ZIP_BASE}_${DIR_NAME}.md5" ]; then | ||
| echo "MD5 file exists for ${DIR_NAME}, skipping" | ||
| continue | ||
| fi | ||
|
|
||
| echo "Zipping ${DUMP_DIR} to ${ZIP_NAME}".7z | ||
| /usr/bin/7z a -p${SECRET} ${ZIP_NAME} -mx=${COMPRESSION_LEVEL} -mhe -t7z ${DUMP_DIR} || { echo "Could not zip ${DUMP_DIR} into ${ZIP_NAME}" ; exit 1; } | ||
| done | ||
|
|
||
| # Create md5 sums for local backups, if they don't exist | ||
| cd ${ZIP_DIR} | ||
| for file in ${ZIP_DIR}/*.7z; do | ||
| # Get the base name of the file without extension | ||
| base_name=$(basename "$file" .7z) | ||
| echo $base_name | ||
| # If a local .md5 file does not exist, create it | ||
| if [ ! -f "${ZIP_DIR}/${base_name}.md5" ]; then | ||
| echo "Local md5 file does not exist for $file, generating, and adding $file to uploads list" | ||
| uploads="$uploads $file" | ||
| local_md5=$(md5sum "$file" | awk '{print $1}') | ||
| echo $local_md5 > "${ZIP_DIR}/${base_name}.md5" | ||
| fi | ||
| done | ||
|
|
||
| # Verify & update list of files to upload | ||
| cd ${ZIP_DIR}/${ZIP_BASE} | ||
| for file in ${ZIP_DIR}/*.7z; do | ||
| # Get the base name of the file without extension | ||
| base_name=$(basename "$file" .7z) | ||
| # Check if the remote md5 file exists | ||
| if [ ! -f "${ZIP_DIR}/tmp_md5/${base_name}.md5" ]; then | ||
| # If the remote md5 file does not exist, add the file to the uploads list | ||
| echo "Remote does not exist for $file, adding $file to uploads list" | ||
| uploads="$uploads $file" | ||
| else | ||
| # Compare local and remote md5 | ||
| remote_md5=$(cat "${ZIP_DIR}/tmp_md5/${base_name}.md5") | ||
| local_md5=$(cat "${ZIP_DIR}/${base_name}.md5") | ||
| if [ "$local_md5" != "$remote_md5" ]; then | ||
| echo "MD5 mismatch for file $file" | ||
| # Extract the last character of the base name | ||
| last_char=${base_name: -1} | ||
| # Check if the last character is a letter | ||
| if [[ $last_char =~ [a-z] ]]; then | ||
| # If it's a letter, increment it | ||
| next_char=$(echo "$last_char" | tr "a-y" "b-z") | ||
| new_base_name=${base_name%?}$next_char | ||
| else | ||
| # If it's not a letter, append 'a' | ||
| new_base_name=${base_name}a | ||
| fi | ||
| # Rename the file | ||
| mv "$file" "${ZIP_DIR}/${new_base_name}.7z" | ||
| # Add the renamed file to the uploads list | ||
| uploads="$uploads ${ZIP_DIR}/${new_base_name}.7z" | ||
| fi | ||
| fi | ||
| echo "Uploads: $uploads" | ||
| done | ||
|
|
||
| ## Verify & update list of files to upload | ||
| #cd ${ZIP_DIR}/${ZIP_BASE} | ||
| #for file in ${ZIP_DIR}/${ZIP_BASE}/*.7z; do | ||
| # # Get the base name of the file without extension | ||
| # base_name=$(basename "$file" .7z) | ||
| # # Check if the remote md5 file exists | ||
| # if [ ! -f "${ZIP_DIR}/${ZIP_BASE}/tmp_md5/${base_name}.md5" ]; then | ||
| # # If the remote md5 file does not exist, add the file to the uploads list | ||
| # echo "Remote does not exist for $file, adding $file to uploads list" | ||
| # uploads="$uploads $file" | ||
| # else | ||
| # # Compare local and remote md5 | ||
| # remote_md5=$(cat "${ZIP_DIR}/${ZIP_BASE}/tmp_md5/${base_name}.md5") | ||
| # local_md5=$(cat "${ZIP_DIR}/${ZIP_BASE}/${base_name}.md5") | ||
| # if [ "$local_md5" != "$remote_md5" ]; then | ||
| # echo "MD5 mismatch for file $file, adding to uploads list" | ||
| # uploads="$uploads $file" | ||
| # fi | ||
| # fi | ||
| # echo "Uploads: $uploads" | ||
| #done | ||
|
|
||
|
|
||
| # Before running rclone | ||
| for file in "${uploads[@]}"; do | ||
| ls $file | ||
| if [ ! -f "$file" ]; then | ||
| echo "File does not exist: $file" | ||
| fi | ||
| done | ||
|
|
||
| ## Before running rclone | ||
| #for file in ${uploads}; do | ||
| # if [ ! -f "$file" ]; then | ||
| # echo "File does not exist: $file" | ||
| # fi | ||
| #done | ||
|
|
||
| ## Sync All Resulting Files (in list!) | ||
| #cd ${ZIP_DIR} | ||
| #for file in "${uploads[@]}"; do | ||
| # echo "RClone-ing ${file} to GCP ${REMOTE}" | ||
| # /usr/bin/rclone sync -v "$file" ${REMOTE}/ | ||
| #done | ||
| #for file in ${uploads}; do | ||
| # echo "RClone-ing ${file} to GCP ${GCP_DEST}" | ||
| # /usr/bin/rclone sync -v "$file" ${REMOTE}/ | ||
| #done | ||
| package main | ||
|
|
||
| import ( | ||
| "os" | ||
| "log" | ||
| // Import other necessary packages | ||
| ) | ||
|
|
||
| func main() { | ||
| // Define the environment variables and their default values | ||
| compressionLevel := getEnv("COMPRESSION_LEVEL", "0") | ||
| deleteDump := getEnv("DELETE_DUMP", "") | ||
| dumpBase := getEnv("DUMP_BASE", "/dump/full_backup") | ||
| dumpRetention := getEnv("DUMP_RETENTION", "3") | ||
| remote := getEnv("REMOTE", "remote:${BUCKET}/${BUCKETPATH}") | ||
| secret := getEnv("SECRET", "") | ||
| slackChannel := getEnv("SLACK_CHANNEL", "") | ||
| slackWebhook := getEnv("SLACK_WEBHOOK", "") | ||
| zipBase := getEnv("ZIP_BASE", "backup_full") | ||
| zipDir := getEnv("ZIP_DIR", "/zip") | ||
| zipRetention := getEnv("ZIP_RETENTION", "4") | ||
|
|
||
| // Check if the necessary files and environment variables exist | ||
| checkFilesAndEnvVars() | ||
|
|
||
| // Delete old zip files and backup dumps based on the retention period | ||
| deleteOldFiles() | ||
|
|
||
| // Get a list of remote backups | ||
| remoteFiles := getRemoteBackups() | ||
|
|
||
| // Create md5 sums for local backups if they don't exist | ||
| createMd5Sums() | ||
|
|
||
| // Verify and update the list of files to upload | ||
| verifyAndUpdateUploads() | ||
|
|
||
| // Sync all resulting files | ||
| syncFiles() | ||
| } | ||
|
|
||
| func getEnv(key, defaultValue string) string { | ||
| value, exists := os.LookupEnv(key) | ||
| if !exists { | ||
| value = defaultValue | ||
| } | ||
| return value | ||
| } | ||
|
|
||
| func checkFilesAndEnvVars() { | ||
| // Implement the function | ||
| } | ||
|
|
||
| func deleteOldFiles() { | ||
| // Implement the function | ||
| } | ||
|
|
||
| func getRemoteBackups() []string { | ||
| // Implement the function | ||
| return []string{} | ||
| } | ||
|
|
||
| func createMd5Sums() { | ||
| // Implement the function | ||
| } | ||
|
|
||
| func verifyAndUpdateUploads() { | ||
| // Implement the function | ||
| } | ||
|
|
||
| func syncFiles() { | ||
| // Implement the function | ||
| } |