Merge pull request #357 from outdoorbits/migrate_to_proftp

allow ftp as backup source to most targets

etc/proftpd_lbb_DefaultRoot.conf

@@ -0,0 +1 @@
+DefaultRoot /media lbb

etc/proftpd_proftpd.conf

@@ -0,0 +1,36 @@
+LoadModule mod_tls.c
+
+SyslogLevel debug
+TransferLog /var/log/proftpd/proftpd.log
+
+<Limit LOGIN>
+    AllowUser pi
+    AllowUser lbb
+    DenyAll
+</Limit>
+
+<Global>
+  DefaultRoot ~/ !lbb
+  Include /etc/proftpd/lbb_DefaultRoot.conf
+</Global>
+
+# Enable FTPS (Explicit TLS)
+<IfModule mod_tls.c>
+  TLSEngine                on
+  TLSRequired              on
+  TLSProtocol              TLSv1.2 TLSv1.3
+  TLSRSACertificateFile    /etc/ssl/certs/little-backup-box.crt
+  TLSRSACertificateKeyFile /etc/ssl/private/little-backup-box.key
+  TLSOptions               NoSessionReuseRequired
+  TLSVerifyClient          off
+</IfModule>
+
+# Disable SFTP in ProFTPD (handled by SSH)
+<IfModule mod_sftp.c>
+  SFTPEngine off
+</IfModule>
+
+# Allow only TLS-protected logins (except local network)
+<IfModule mod_auth.c>
+  RequireValidShell off
+</IfModule>

install-little-backup-box.sh

@@ -169,13 +169,13 @@ sudo DEBIAN_FRONTEND=noninteractive \
 		-o "Dpkg::Options::=--force-confold" \
 		-o "Dpkg::Options::=--force-confdef" \
 		install -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages \
-		acl git screen rsync exfat-fuse exfatprogs ntfs-3g acl bindfs gphoto2 libimage-exiftool-perl php php-cli samba samba-common-bin vsftpd imagemagick curl libimobiledevice6 ifuse sshpass f3 sqlite3 php-sqlite3 ffmpeg libheif-examples libraw-bin openvpn wireguard openresolv hfsprogs fuse3 python3 python3-pip python3-pil python3-configobj python3-gpiozero python3-rpi-lgpio python3-qrcode python3-psutil smartmontools dos2unix
+		acl git screen rsync exfat-fuse exfatprogs ntfs-3g acl bindfs gphoto2 libimage-exiftool-perl php php-cli samba samba-common-bin proftpd-basic proftpd-mod-crypto imagemagick curl libimobiledevice6 ifuse sshpass f3 sqlite3 php-sqlite3 ffmpeg libheif-examples libraw-bin openvpn wireguard openresolv hfsprogs fuse3 python3 python3-pip python3-pil python3-configobj python3-gpiozero python3-rpi-lgpio python3-qrcode python3-psutil smartmontools dos2unix
 
 # Remove packages not needed anymore
 if [ "${SCRIPT_MODE}" = "update" ]; then
 	echo "apt-get purge..."
 	sudo DEBIAN_FRONTEND=noninteractive \
-		apt-get purge minidlna -y
+		apt-get purge minidlna vsftpd -y
 fi
 
 # Remove obsolete packages
@@ -557,16 +557,13 @@ sudo a2ensite little-backup-box
 yes | sudo cp -f "${INSTALLER_DIR}/etc/samba_smb.conf" "/etc/samba/smb.conf"
 sudo chmod 0440 "/etc/samba/smb.conf"
 
-# Configure vsftpd
-yes | sudo cp -f "${INSTALLER_DIR}/etc/vsftpd.conf" "/etc/vsftpd.conf"
-sudo chmod 0440 "/etc/vsftpd.conf.conf"
+# Configure proftpd
+yes | sudo cp -f "${INSTALLER_DIR}/etc/proftpd_proftpd.conf" "/etc/proftpd/proftpd.conf"
+yes | sudo cp -f "${INSTALLER_DIR}/etc/proftpd_lbb_DefaultRoot.conf" "/etc/proftpd/lbb_DefaultRoot.conf"
+sudo chmod 644 "/etc/proftpd/proftpd.conf"
+sudo chmod 644 "/etc/proftpd/lbb_DefaultRoot.conf"
 
-sudo useradd -s /bin/false -r ftpsecure
-
-sudo mkdir -p /var/run/vsftpd/empty
-sudo chown ftpsecure /var/run/vsftpd/empty
-
-sudo service vsftpd restart
+sudo service proftpd restart
 
 # setup graphical environment
 source "${INSTALLER_DIR}/setup-graphical-environment.sh"

scripts/backup.py

@@ -51,7 +51,7 @@ class backup(object):
 
 	def __init__(self, SourceName, TargetName, move_files='setup', DoRenameFiles='setup', ForceSyncDatabase=False, DoGenerateThumbnails='setup', shiftGenerateThumbnails=False, DoUpdateEXIF='setup', DeviceIdentifierPresetSource=None, DeviceIdentifierPresetTarget=None, PowerOff='setup', SecondaryBackupFollows=False):
 
-		# SourceName:											one of ['anyusb', 'usb', 'internal', 'nvme', 'camera', 'cloud:SERVICE_NAME', 'cloud_rsync'] or functions: ['thumbnails', 'database', 'exif', 'rename]
+		# SourceName:											one of ['anyusb', 'usb', 'internal', 'nvme', 'camera', 'cloud:SERVICE_NAME', 'cloud_rsync', 'ftp'] or functions: ['thumbnails', 'database', 'exif', 'rename]
 		# TargetName:											one of ['anyusb', 'usb', 'internal', 'nvme', 'cloud:SERVICE_NAME', 'cloud_rsync']
 		# DoRenameFiles, DoGenerateThumbnails, DoUpdateEXIF:	one of ['setup', True, False]
 		# ForceSyncDatabase:									one of [True, False]
@@ -149,6 +149,8 @@ def __init__(self, SourceName, TargetName, move_files='setup', DoRenameFiles='se
 			except:
 				pass
 
+		self.TransferMode	= None if self.SourceStorageType == 'ftp' else self.TransferMode
+
 		# Unmount devices, clean before backup
 		lib_storage.umount(self.__setup,'all')
 
@@ -228,6 +230,10 @@ def run(self):
 		self.finish()
 
 	def get_syncCommand(self, TransferMode, SubPathAtSource, dry_run=False):
+
+		if TransferMode is None:
+			return([])
+
 		syncCommand	= []
 
 		## excludes
@@ -300,6 +306,10 @@ def get_syncCommand(self, TransferMode, SubPathAtSource, dry_run=False):
 		return(syncCommand)
 
 	def calculate_files_to_sync(self, singleSubPathsAtSource=None):
+
+		if self.TransferMode is None:
+			return(0, True)
+
 		if singleSubPathsAtSource:
 			checkPathsList	= [singleSubPathsAtSource]
 		else:
@@ -435,6 +445,10 @@ def backup(self):
 					SourceStorageName	= 'usb' if self.SourceName == 'anyusb' else self.SourceName
 					SourceStorageType	= SourceStorageName
 
+				if self.SourceName =='ftp':
+					SourceStorageName	= 'ftp'
+					SourceStorageType	= SourceStorageName
+
 				if todoSources:
 					Identifier	= todoSources[0]
 				elif Identifier_OLD:
@@ -448,18 +462,29 @@ def backup(self):
 			# Set the PWR LED to blink long to indicate waiting for the source device
 			lib_system.rpi_leds(trigger='timer',delay_on=750,delay_off=250)
 
-			if SourceStorageType in ['usb', 'internal','nvme', 'camera', 'cloud', 'cloud_rsync']:
-				self.SourceDevice	= lib_storage.storage(StorageName=SourceStorageName, Role=lib_storage.role_Source, WaitForDevice=True, DeviceIdentifierPresetThis=Identifier, DeviceIdentifierPresetOther=self.TargetDevice.DeviceIdentifier)
+			if SourceStorageType in ['usb', 'internal','nvme', 'camera', 'cloud', 'cloud_rsync', 'ftp']:
+				self.SourceDevice	= lib_storage.storage(StorageName=SourceStorageName, Role=lib_storage.role_Source, WaitForDevice=True, DeviceIdentifierPresetThis=Identifier, DeviceIdentifierPresetOther=self.TargetDevice.DeviceIdentifier, PartnerDevice=self.TargetDevice)
 
-				self.__display.message([f":{self.__lan.l('box_backup_mounting_target')}", f":{self.__lan.l(f'box_backup_mode_{self.TargetDevice.StorageType}')} {self.TargetDevice.CloudServiceName}"])
+				self.__display.message([f":{self.__lan.l('box_backup_mounting_source')}", f":{self.__lan.l(f'box_backup_mode_{self.SourceDevice.StorageType}')} {self.SourceDevice.CloudServiceName}"])
 				self.SourceDevice.mount()
 
+
 			elif SourceStorageType in ['thumbnails', 'database', 'exif']:
 				pass
 			else:
 				self.__display.message([f":{self.__lan.l('box_backup_invalid_mode_combination_1')}", f":{self.__lan.l('box_backup_invalid_mode_combination_2')}", f":{self.__lan.l('box_backup_invalid_mode_combination_3')}"])
 				return()
 
+			# for ftp source, the job is done now.
+			if SourceStorageType	== 'ftp':
+				while self.TargetDevice.mounted():
+					time.sleep(5)
+
+				self.SourceDevice.umount()
+				self.__display.message([f":{self.__lan.l('box_backup_break1')}", f":{self.__lan.l('box_backup_break2')}"])
+
+				sys.exit()
+
 			# remember SourceStorageName for next run
 			if SourceStorageName=='camera':
 				Identifier_OLD	= lib_storage.format_CameraIdentifier(self.SourceDevice.DeviceIdentifier, self.SourceDevice.CameraPort)
@@ -1400,7 +1425,7 @@ def __cleanup(self):
 		epilog		= 'This script can ideally be configured and started via the Little Backup Box web UI.'
 	)
 
-	SourceChoices	= ['anyusb', 'usb', 'internal', 'nvme', 'camera'] + CloudServices + ['cloud_rsync', 'thumbnails', 'database', 'exif', 'rename']
+	SourceChoices	= ['anyusb', 'usb', 'internal', 'nvme', 'camera'] + CloudServices + ['cloud_rsync', 'ftp', 'thumbnails', 'database', 'exif', 'rename']
 	parser.add_argument(
 		'--SourceName',
 		'-s',

scripts/constants.sh

@@ -12,6 +12,7 @@ const_MOUNTPOINT_CLOUD_SOURCE="cloud_source"
 const_MOUNT_LOCAL_TIMEOUT=30
 const_MOUNT_CLOUD_TIMEOUT=30
 const_INTERNAL_BACKUP_DIR="internal"
+const_LBB_FTP_BACKUP_SUB_DIR="LBB-FTP"
 const_BACKGROUND_IMAGES_DIR="backgroundimages"
 const_WEB_ROOT_LBB="/var/www/little-backup-box"
 const_LOGFILE="/var/www/little-backup-box/tmp/little-backup-box.log"

scripts/cron_idletime.py

@@ -31,9 +31,9 @@ class idletime(object):
 	def __init__(self):
 		#definitions
 		self.WORKING_DIR	= os.path.dirname(__file__)
-		self.ApacheAccessLogfile		= '/var/log/apache2/lbb-access.log'
-		self.ApacheRcloneAccessLogfile	= '/var/log/apache2/rclone-access.log'
-		self.vsftpdAccessLogfile				= '/var/log/vsftpd.log'
+		self.ApacheAccessLogfile				= '/var/log/apache2/lbb-access.log'
+		self.ApacheRcloneAccessLogfile			= '/var/log/apache2/rclone-access.log'
+		self.proftpdAccessLogfile				= '/var/log/proftpd/proftpd.log'
 
 		#objects
 		self.__setup	= lib_setup.setup()
@@ -73,10 +73,10 @@ def check(self):
 		if ApacheRcloneLogfileAgeSec < IdleSecToPowerOff:
 			return(f'idletime: logfile rclone gui idletime not reached ({ApacheRcloneLogfileAgeSec}s < {IdleSecToPowerOff}s)')
 
-		# logfile vsftpd
-		vsftpdLogfileAgeSec			= CompareTime - os.stat(self.vsftpdAccessLogfile).st_mtime if os.path.isfile(self.vsftpdAccessLogfile) else IdleSecToPowerOff
-		if vsftpdLogfileAgeSec < IdleSecToPowerOff:
-			return(f'idletime: logfile vsftpd idletime not reached ({vsftpdLogfileAgeSec}s < {IdleSecToPowerOff}s)')
+		# logfile proftpd
+		proftpdLogfileAgeSec			= CompareTime - os.stat(self.proftpdAccessLogfile).st_mtime if os.path.isfile(self.proftpdAccessLogfile) else IdleSecToPowerOff
+		if proftpdLogfileAgeSec < IdleSecToPowerOff:
+			return(f'idletime: logfile proftpd idletime not reached ({proftpdLogfileAgeSec}s < {IdleSecToPowerOff}s)')
 
 		# check processes
 		for process in [

scripts/index.php

@@ -69,6 +69,8 @@
 		$CloudServices_marked[]	= 'cloud:' . $CloudService;
 	}
 
+	$LocalNetworkServices	= array('ftp');
+
 	// rsync
 	$rsync_configurated	= ($config['conf_RSYNC_SERVER']=='' or $config['conf_RSYNC_PORT']=='' or $config['conf_RSYNC_USER']=='' or $config['conf_RSYNC_PASSWORD']=='' or $config['conf_RSYNC_SERVER_MODULE']=='') == false;
 	if ($rsync_configurated) {
@@ -79,7 +81,7 @@
 		'anyusb'	=> $LocalAutoServices,
 		'usb'		=> $LocalServices,
 		'camera'	=> $CameraServices,
-		'cloud'		=> $CloudServices_marked
+		'cloud'		=> array_merge($CloudServices_marked, $LocalNetworkServices)
 	);
 
 	$TargetServices			= array(
@@ -113,7 +115,8 @@ function HideDisallowedButtons(ActiveSource) {
 				if (
 					((TargetService === ActiveSource.value) && (TargetService !== 'usb')) ||
 					((ActiveSource.value === 'anyusb') && (TargetService === 'cloud_rsync')) ||
-					((ActiveSource.value === 'camera') && (TargetService === 'cloud_rsync'))
+					((ActiveSource.value === 'camera') && (TargetService === 'cloud_rsync')) ||
+					((ActiveSource.value === 'ftp') && (TargetService === 'cloud_rsync'))
 				) {
 					document.getElementById("Target_" + TargetService).disabled = true;
 				} else {
@@ -173,6 +176,10 @@ function HideDisallowedButtons(ActiveSource) {
 									elseif ($LabelName == 'cloud_rsync') {
 										$LabelName		= l::box_backup_mode_cloud_rsync;
 									}
+									elseif ($LabelName == 'ftp') {
+										$LabelName		= l::box_backup_mode_ftp;
+									}
+
 									print("<input type='radio' name='SourceDevice' value='$Storage' id='Source_$Storage' onchange='HideDisallowedButtons(this)' " . ($Storage == $OldSource ? 'checked' : '') . ">");
 									print("<label for='Source_$Storage'>$LabelName</label></br>");
 								}

scripts/lang/de.json

@@ -516,7 +516,8 @@
         "usbs": "USB-Speicher",
         "nvme": "NVMe SSD",
         "nvmes": "NVMe SSDs",
-        "rename": "Umbenennen"
+        "rename": "Umbenennen",
+        "ftp": "LBB's FTP-Server"
       },
       "nvme": "NVMe SSD",
       "nvme_source": "USB-Quelle",
@@ -534,6 +535,9 @@
       "mounting": {
         "source": "Quelle einbinden",
         "target": "Ziel einbinden"
+      },
+      "ftp": {
+        "started": "FTP-Server gestartet."
       }
     },
     "cronip": {

scripts/lang/en.json

@@ -122,7 +122,8 @@
         "usbs": "USB storages",
         "nvme": "NVMe SSD",
         "nvmes": "NVMe SSDs",
-        "rename": "Rename"
+        "rename": "Rename",
+        "ftp": "LBB's FTP server"
       },
       "nvme": "NVMe SSD",
       "nvme_source": "USB-source",
@@ -140,6 +141,9 @@
       "mounting": {
         "source": "Mount source",
         "target": "Mount target"
+      },
+      "ftp": {
+        "started": "FTP server started."
       }
     },
     "poweroff": {

scripts/lang/es.json

@@ -516,7 +516,8 @@
         "usbs": "Almacenamientos USB",
         "nvme": "NVMe SSD",
         "nvmes": "NVMe SSDs",
-        "rename": "Rebautizar"
+        "rename": "Rebautizar",
+        "ftp": "Servidor FTP de LBB"
       },
       "nvme": "NVMe SSD",
       "nvme_source": "Fuente USB",
@@ -534,6 +535,9 @@
       "mounting": {
         "source": "Integrar fuente",
         "target": "Integrar objetivo"
+      },
+      "ftp": {
+        "started": "Servidor FTP iniciado."
       }
     },
     "cronip": {

scripts/lang/fr.json

@@ -516,7 +516,8 @@
         "usbs": "Stockages USB",
         "nvme": "NVMe SSD",
         "nvmes": "NVMe SSDs",
-        "rename": "Rebaptiser"
+        "rename": "Rebaptiser",
+        "ftp": "Serveur FTP de LBB"
       },
       "nvme": "NVMe SSD",
       "nvme_source": "Source USB",
@@ -534,6 +535,9 @@
       "mounting": {
         "source": "Monter la source",
         "target": "Monter la cible"
+      },
+      "ftp": {
+        "started": "Le serveur FTP a démarré."
       }
     },
     "cronip": {