diff --git a/config/rclone/rclone.conf.example b/config/rclone/rclone.conf.example new file mode 100644 index 00000000..0ecf0c59 --- /dev/null +++ b/config/rclone/rclone.conf.example @@ -0,0 +1,82 @@ +# ============================================================================= +# Rclone Configuration Example +# ============================================================================= +# This is an example rclone configuration file. +# Copy this to config/rclone/rclone.conf on your host and customize it. +# +# Generate a new config with: rclone config +# Docs: https://rclone.org/docs/ +# ============================================================================= + +# ----------------------------------------------------------------------------- +# S3-Compatible Storage (AWS S3, MinIO, Backblaze B2, etc.) +# ----------------------------------------------------------------------------- +[s3-backup] +type = s3 +provider = AWS +access_key_id = YOUR_AWS_ACCESS_KEY +secret_access_key = YOUR_AWS_SECRET_KEY +region = us-east-1 +bucket = your-backup-bucket +acl = private +# For MinIO or other S3-compatible services: +# provider = Other +# endpoint = https://your-minio-server:9000 +# access_key_id = YOUR_MINIO_ACCESS_KEY +# secret_access_key = YOUR_MINIO_SECRET_KEY + +# ----------------------------------------------------------------------------- +# Google Drive +# ----------------------------------------------------------------------------- +[gdrive-backup] +type = drive +client_id = YOUR_GOOGLE_CLIENT_ID +client_secret = YOUR_GOOGLE_CLIENT_SECRET +token = {"access_token":"YOUR_ACCESS_TOKEN","token_type":"Bearer","refresh_token":"YOUR_REFRESH_TOKEN","expiry":"YOUR_EXPIRY_TIME"} +drive_id = YOUR_DRIVE_ID +root_folder_id = YOUR_ROOT_FOLDER_ID +# For shared drive/team drive: +# team_drive = YOUR_TEAM_DRIVE_ID + +# ----------------------------------------------------------------------------- +# OneDrive +# ----------------------------------------------------------------------------- +[onedrive-backup] +type = onedrive +client_id = YOUR_ONEDRIVE_CLIENT_ID +client_secret = YOUR_ONEDRIVE_CLIENT_SECRET +tenant = common +drive_type = personal +# For OneDrive for Business: +# drive_type = business + +# ----------------------------------------------------------------------------- +# Backblaze B2 +# ----------------------------------------------------------------------------- +[b2-backup] +type = b2 +account = YOUR_B2_ACCOUNT_ID +key = YOUR_B2_APPLICATION_KEY +bucket = your-b2-bucket +# For B2 with encryption: +# encryption = AES-256 + +# ----------------------------------------------------------------------------- +# Mega.nz +# ----------------------------------------------------------------------------- +[mega-backup] +type = mega +user = your@mega.email +password = YOUR_MEGA_PASSWORD + +# ----------------------------------------------------------------------------- +# SFTP/SSH +# ----------------------------------------------------------------------------- +[sftp-backup] +type = sftp +host = your-sftp-server.com +user = your-username +port = 22 +key_file = /path/to/private/key +# key_file_pass = YOUR_KEY_PASSWORD +known_hosts_file = ~/.ssh/known_hosts diff --git a/scripts/backup-to-cloud.sh b/scripts/backup-to-cloud.sh new file mode 100644 index 00000000..9acf7e6d --- /dev/null +++ b/scripts/backup-to-cloud.sh @@ -0,0 +1,231 @@ +#!/bin/bash +# ============================================================================= +# HomeLab Stack — Backup to Cloud Script +# ============================================================================= +# Syncs local backups from /opt/homelab-backups to cloud storage via Rclone +# +# Usage: +# ./scripts/backup-to-cloud.sh # Interactive mode +# ./scripts/backup-to-cloud.sh --dry-run # Test without making changes +# ./scripts/backup-to-cloud.sh s3-backup # Sync to specific remote +# ./scripts/backup-to-cloud.sh gdrive-backup # Sync to Google Drive +# +# Cron example (daily at 2 AM): +# 0 2 * * * /opt/homelab-stack/scripts/backup-to-cloud.sh >> /var/log/backup-to-cloud.log 2>&1 +# +# Payment: https://lll.io/bounty/0xaae0101ac77a2e4e0ea826eb4d309374f029b0a6 +# ============================================================================= + +set -euo pipefail + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(dirname "$SCRIPT_DIR")" +BACKUP_SOURCE="${BACKUP_SOURCE:-/opt/homelab-backups}" +RCLONE_CONFIG="${RCLONE_CONFIG:-$ROOT_DIR/config/rclone/rclone.conf}" +LOG_FILE="${LOG_FILE:-/var/log/backup-to-cloud.log}" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" | tee -a "$LOG_FILE" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$LOG_FILE" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$LOG_FILE" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" | tee -a "$LOG_FILE" +} + +# Show usage +usage() { + cat << EOF +Usage: $(basename "$0") [OPTIONS] [REMOTE] + +Sync local backups to cloud storage using Rclone. + +Arguments: + REMOTE Rclone remote name to sync to (e.g., s3-backup, gdrive-backup) + If not provided, lists available remotes. + +Options: + --dry-run Show what would be transferred without making changes + --verbose Show detailed output + --help Show this help message + +Examples: + $(basename "$0") # List available remotes + $(basename "$0") --dry-run s3-backup # Test sync to S3 + $(basename "$0") gdrive-backup # Sync to Google Drive + $(basename "$0") b2-backup # Sync to Backblaze B2 + +Environment Variables: + BACKUP_SOURCE Source directory (default: /opt/homelab-backups) + RCLONE_CONFIG Path to rclone.conf (default: ./config/rclone/rclone.conf) + LOG_FILE Log file path (default: /var/log/backup-to-cloud.log) + +EOF +} + +# Check prerequisites +check_prereqs() { + if ! command -v rclone &> /dev/null; then + log_error "rclone not found. Please install rclone first." + log_info "Install: https://rclone.org/install/" + exit 1 + fi + + if [ ! -f "$RCLONE_CONFIG" ]; then + log_error "Rclone config not found at: $RCLONE_CONFIG" + log_info "Please copy config/rclone/rclone.conf.example to config/rclone/rclone.conf" + exit 1 + fi + + if [ ! -d "$BACKUP_SOURCE" ]; then + log_warning "Backup source directory does not exist: $BACKUP_SOURCE" + log_info "Creating directory..." + mkdir -p "$BACKUP_SOURCE" + fi +} + +# List available remotes +list_remotes() { + log_info "Available Rclone remotes:" + rclone listremotes --config "$RCLONE_CONFIG" 2>/dev/null | while read -r remote; do + echo " - ${remote%/}" + done +} + +# List available remotes and exit +show_remotes_and_exit() { + check_prereqs + echo "" + log_info "Available cloud backup targets:" + list_remotes + echo "" + log_info "Usage: $(basename "$0") [remote-name]" + exit 0 +} + +# Sync backup to remote +sync_to_remote() { + local remote="$1" + local dry_run="${DRY_RUN:-}" + local verbose="${VERBOSE:-}" + + # Validate remote exists + if ! rclone listremotes --config "$RCLONE_CONFIG" | grep -q "^${remote}:$"; then + log_error "Remote '$remote' not found in rclone config." + log_info "Available remotes:" + list_remotes + exit 1 + fi + + # Check source has content + if [ -z "$(ls -A "$BACKUP_SOURCE" 2>/dev/null)" ]; then + log_warning "Backup source directory is empty: $BACKUP_SOURCE" + log_info "Nothing to sync." + exit 0 + fi + + # Build rclone command + local cmd="rclone sync \"$BACKUP_SOURCE\" \"${remote}:\" --config \"$RCLONE_CONFIG\"" + + [ -n "$dry_run" ] && cmd="$cmd --dry-run" + [ -n "$verbose" ] && cmd="$cmd --verbose" + [ -n "$verbose" ] && cmd="$cmd -v" + + cmd="$cmd --log-file \"$LOG_FILE\"" + cmd="$cmd --log-level INFO" + cmd="$cmd --stats 1s" + cmd="$cmd --stats-one-line" + cmd="$cmd --transfers 4" + cmd="$cmd --checkers 8" + cmd="$cmd --bwlimit 10M" + cmd="$cmd --exclude \"*.tmp\"" + cmd="$cmd --exclude \"*.part\"" + cmd="$cmd --exclude \".*\"" # Exclude hidden files + + echo "" + log_info "Starting backup sync to ${remote}:" + log_info " Source: $BACKUP_SOURCE" + log_info " Remote: ${remote}:" + log_info " Config: $RCLONE_CONFIG" + [ -n "$dry_run" ] && log_warning "DRY-RUN MODE - No changes will be made" + echo "" + + # Execute sync + if eval "$cmd"; then + log_success "Backup sync completed successfully!" + + # Show stats + if [ -z "$dry_run" ]; then + echo "" + log_info "Sync summary:" + rclone about "${remote}:" --json 2>/dev/null | jq -r '.used, .free, .total' 2>/dev/null || true + fi + else + log_error "Backup sync failed!" + exit 1 + fi +} + +# Main script +main() { + # Parse arguments + DRY_RUN="" + VERBOSE="" + REMOTE="" + + while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + DRY_RUN="1" + shift + ;; + --verbose|-v) + VERBOSE="1" + shift + ;; + --help|-h) + usage + exit 0 + ;; + --*) + log_error "Unknown option: $1" + usage + exit 1 + ;; + *) + REMOTE="$1" + shift + ;; + esac + done + + # Check prerequisites + check_prereqs + + # If no remote specified, show available remotes + if [ -z "$REMOTE" ]; then + show_remotes_and_exit + fi + + # Sync to specified remote + sync_to_remote "$REMOTE" +} + +# Run main function +main "$@" diff --git a/stacks/backup/.env.example b/stacks/backup/.env.example new file mode 100644 index 00000000..50ddfbff --- /dev/null +++ b/stacks/backup/.env.example @@ -0,0 +1,41 @@ +# ============================================================================= +# HomeLab Stack — Backup Stack Environment Configuration +# Copy this file to .env in the stacks/backup directory +# ============================================================================= + +# ----------------------------------------------------------------------------- +# GENERAL +# ----------------------------------------------------------------------------- +TZ=Asia/Shanghai +PUID=1000 +PGID=1000 +DOMAIN=yourdomain.com # Your base domain (e.g. home.example.com) + +# ----------------------------------------------------------------------------- +# RESTIC REST SERVER +# ----------------------------------------------------------------------------- +RESTIC_PASSWORD= # REQUIRED: Password for restic repository + # Generate with: openssl rand -base64 32 + +# ----------------------------------------------------------------------------- +# RCLONE (Cloud Backup Targets) +# ----------------------------------------------------------------------------- +# Create rclone.conf with your cloud storage configs: +# [s3-backup] +# type = s3 +# provider = AWS +# access_key_id = YOUR_AWS_KEY +# secret_access_key = YOUR_AWS_SECRET +# region = us-east-1 +# bucket = your-backup-bucket +# +# [gdrive-backup] +# type = drive +# client_id = YOUR_GOOGLE_CLIENT_ID +# client_secret = YOUR_GOOGLE_CLIENT_SECRET +# token = {"access_token":"..."} +# drive_id = YOUR_DRIVE_ID +# root_folder_id = YOUR_FOLDER_ID +# +# NOTE: Mount your rclone.conf at stacks/backup/config/rclone/rclone.conf +# and ensure it has proper permissions: chmod 600 rclone.conf diff --git a/stacks/backup/docker-compose.yml b/stacks/backup/docker-compose.yml index 1e38d381..915bd823 100644 --- a/stacks/backup/docker-compose.yml +++ b/stacks/backup/docker-compose.yml @@ -1,4 +1,22 @@ +# ============================================================================= +# HomeLab Stack — Backup & Disaster Recovery Stack +# Services: Duplicati, Restic REST Server, Rclone +# +# Prerequisites: +# 1. cp .env.example .env && fill in required values +# 2. Ensure proxy network exists: docker network create proxy +# 3. Ensure backup directories exist: mkdir -p /opt/homelab-backups /opt/restic-data +# 4. docker compose up -d +# +# Payment: https://lll.io/bounty/0xaae0101ac77a2e4e0ea826eb4d309374f029b0a6 +# ============================================================================= + services: + # --------------------------------------------------------------------------- + # Duplicati — Backup Client with Web UI + # Access: https://duplicati.${DOMAIN} + # Docs: https://duplicati.readthedocs.io/ + # --------------------------------------------------------------------------- duplicati: image: lscr.io/linuxserver/duplicati:2.0.8 container_name: duplicati @@ -8,23 +26,31 @@ services: volumes: - duplicati-config:/config - duplicati-data:/data - - /opt/homelab-backups:/backups + - duplicati-backups:/backups + - /opt/homelab:/source:ro # Mount your data paths here environment: - TZ=${TZ:-Asia/Shanghai} - PUID=${PUID:-1000} - PGID=${PGID:-1000} labels: - - traefik.enable=true + - "traefik.enable=true" - "traefik.http.routers.duplicati.rule=Host(`duplicati.${DOMAIN}`)" - - traefik.http.routers.duplicati.entrypoints=websecure - - traefik.http.routers.duplicati.tls=true - - traefik.http.services.duplicati.loadbalancer.server.port=8200 + - "traefik.http.routers.duplicati.entrypoints=websecure" + - "traefik.http.routers.duplicati.tls.certresolver=letsencrypt" + - "traefik.http.routers.duplicati.middlewares=security-headers@file" + - "traefik.http.services.duplicati.loadbalancer.server.port=8200" healthcheck: test: ["CMD-SHELL", "wget --spider -q http://localhost:8200 || exit 1"] interval: 30s timeout: 10s retries: 3 + start_period: 60s + # --------------------------------------------------------------------------- + # Restic REST Server — S3-Compatible Backup Repository + # Access: https://restic.${DOMAIN} + # Docs: https://github.com/restic/rest-server + # --------------------------------------------------------------------------- restic-rest-server: image: restic/rest-server:0.13.0 container_name: restic-backup-server @@ -36,24 +62,76 @@ services: environment: - TZ=${TZ:-Asia/Shanghai} - RESTIC_REST_SERVER_PASSWORD=${RESTIC_PASSWORD:-} - command: --listen ":8080" --path /data + - RESTIC_REST_SERVER_LISTEN=:8080 + command: --path /data --exclusive-create labels: - - traefik.enable=true + - "traefik.enable=true" - "traefik.http.routers.restic.rule=Host(`restic.${DOMAIN}`)" - - traefik.http.routers.restic.entrypoints=websecure - - traefik.http.routers.restic.tls=true - - traefik.http.services.restic.loadbalancer.server.port=8080 + - "traefik.http.routers.restic.entrypoints=websecure" + - "traefik.http.routers.restic.tls.certresolver=letsencrypt" + - "traefik.http.routers.restic.middlewares=security-headers@file" + - "traefik.http.services.restic.loadbalancer.server.port=8080" healthcheck: test: ["CMD-SHELL", "wget --spider -q http://localhost:8080 || exit 1"] interval: 30s timeout: 10s retries: 3 + start_period: 30s + + # --------------------------------------------------------------------------- + # Rclone — Cloud Backup Sync + # Mounts and syncs cloud storage providers (S3, Google Drive, etc.) + # Config: /config/rclone/rclone.conf + # Docs: https://rclone.org/ + # --------------------------------------------------------------------------- + rclone: + image: rclone/rclone:1.68 + container_name: rclone + restart: unless-stopped + networks: + - proxy + volumes: + - rclone-config:/config/rclone:ro + - /opt/homelab-backups:/data:ro + environment: + - TZ=${TZ:-Asia/Shanghai} + - RCLONE_CONFIG=/config/rclone/rclone.conf + command: --rc --rc-addr 0.0.0.0:5572 --log-level INFO serve restic /data --args "--bits 16" + labels: + - "traefik.enable=true" + - "traefik.http.routers.rclone.rule=Host(`rclone.${DOMAIN}`)" + - "traefik.http.routers.rclone.entrypoints=websecure" + - "traefik.http.routers.rclone.tls.certresolver=letsencrypt" + - "traefik.http.routers.rclone.middlewares=security-headers@file" + - "traefik.http.services.rclone.loadbalancer.server.port=5572" + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://localhost:5572 || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s +# ============================================================================= +# Networks +# NOTE: 'proxy' network is EXTERNAL — create it before first run: +# docker network create proxy +# All other stacks join this network to be accessible via Traefik. +# ============================================================================= networks: proxy: external: true +# ============================================================================= +# Volumes +# ============================================================================= volumes: duplicati-config: + driver: local duplicati-data: + driver: local + duplicati-backups: + driver: local restic-data: + driver: local + rclone-config: + driver: local