Merge branch 'master' of https://github.com/emoncms/backup

Conflicts: emoncms-export.sh
emoncms · Oct 28, 2018 · e811075 · e811075
2 parents d901189 + 923d3cd
commit e811075
Show file tree

Hide file tree

Showing 9 changed files with 138 additions and 94 deletions.
diff --git a/backup/backup_controller.php b/backup/backup_controller.php
@@ -14,7 +14,7 @@
 
 function backup_controller()
 {
-    global $route, $session, $path;
+    global $route, $session, $path, $redis;
     $result = false;
 
     $export_flag = "/tmp/emoncms-flag-export";
@@ -34,14 +34,8 @@ function backup_controller()
 
     if ($route->action == 'start') {
         $route->format = "text";
-        $fh = @fopen($export_flag,"w");
-        if (!$fh) {
-            $result = "ERROR: Can't write the flag $export_flag.";
-        } else {
-            fwrite($fh,"$export_script>$export_logfile");
-            $result = "Backup flag set";
-        }
-        @fclose($fh);
+
+        $redis->rpush("service-runner","$export_script $export_flag>$export_logfile");
     }
 
     if ($route->action == 'exportlog') {
@@ -87,15 +81,7 @@ function backup_controller()
 
         if ((move_uploaded_file($_FILES['file']['tmp_name'], $target_path)) && ($uploadOk == 1)) {
 
-            $fh = @fopen($import_flag,"w");
-            if (!$fh) {
-                $result = "ERROR: Can't write the flag $import_flag.";
-            } else {
-                fwrite($fh,"$import_script>$import_logfile");
-                $result = "Backup flag set";
-            }
-            @fclose($fh);
-
+            $redis->rpush("service-runner","$import_script $import_flag>$import_logfile");
             header('Location: '.$path.'backup');
         } else {
             $result = "Sorry, there was an error uploading the file";

diff --git a/backup/backup_view.php b/backup/backup_view.php
@@ -30,43 +30,58 @@
 
 <br>
 <h2>Backup</h2>
+<hr>
 <table class="table">
 <tr>
-    <td>
-        <h3>Export</h3>
-        <p>Export a compressed archive containing the emoncms mysql database, phpfina, phptimeseries data files, emonhub.conf and emoncms.conf. This can be used to migrate data to another emonpi or emonbase. Refresh page to see archive download link once export is complete.</p>
-
-        <pre id="export-log-bound"><div id="export-log"></div></pre>
-    </td>
-    <td class="buttons"><br>
-        <button id="emonpi-backup" class="btn btn-info"><?php echo _('Create backup'); ?></button>
-        <?php
-        $backup_filename="emoncms-backup-".date("Y-m-d").".tar.gz";
-        if (file_exists("/home/pi/data/$backup_filename") && !file_exists("/tmp/backuplock"))
-        {
-            echo '<br><br><b>Right Click > Download:</b><br><a href="'.$path.'backup/download">'.$backup_filename.'</a>';
-        }
-        ?>
-    </td>
+  <h3>Export</h3>
+  <p>Export a compressed archive containing:</p>
+  <ul>
+    <li>Emoncms MYSQL database</li>
+    <li>PHPFINA data files</li>
+    <li>PHPTIMESERIES data files</li>
+    <li>EmonHub Config</li>
+    <li>Emoncms Config</li>
+  </ul>
+  <p>These files contain all Emoncms data including:</p>
+  <ul>
+    <li>Input processes</li>
+    <li>Feed data</li>
+    <li>Dashboards</li>
+    <li>EmonHub Config</li>
+  </ul>
+  <p>The compressed archive can be used to migrate data to another emonPi / emonBase.</p>
+  <button id="emonpi-backup" class="btn btn-info"><?php echo _('Create backup'); ?></button>
+  <br><br>
+  <pre id="export-log-bound"><div id="export-log"></div></pre>
+  <?php
+  $backup_filename="emoncms-backup-".date("Y-m-d").".tar.gz";
+  if (file_exists("/home/pi/data/$backup_filename") && !file_exists("/tmp/backuplock"))
+  {
+      echo '<br><br><b>Right Click > Download:</b><br><a href="'.$path.'backup/download">'.$backup_filename.'</a>';
+  }
+  ?>
+  <br><br>
+  <p>Once export is complete refresh page to see download link.</p>
+
+  <p><i>Note: Export can take a long some time, please be patient.</i></p>
+
 </tr>
-
+<hr>
 <tr>
-    <td>
-        <h3>Import</h3>
-        <p>Import an emoncms backup archive containing the emoncms mysql database, phpfina, phptimeseries data files, emonhub.conf and emoncms.conf.</p>
-        <p>Before import ensure latest version of Emoncms & emonHub.</p>
-	<p><b>*CAUTION ALL EMONCMS ACCOUNT DATA WILL BE OVERWRITTEN BY THE IMPORTED DATA*</b></p>
-	<p>Note: If browser upload fails for large backup files <a href="http://github.com/emoncms/backup">follow manual import instructions.</a> </p>
-        <pre id="import-log-bound"><div id="import-log"></div></pre>
-        <br>
-        <p>Refresh page if log window is not updating. After import is complete logout then log back in using the imported account login details.</p>
-    </td>
-    <td>
-        <form action="<?php echo $path; ?>backup/upload" method="post" enctype="multipart/form-data">
-        <input type="file" name="file" id="file"><br><br>
-        <input class="btn btn-danger" type="submit" name="submit" value="Import Backup">
-        </form>
-    </td>
+  <h3>Import</h3>
+  <p>Import an emoncms backup archive.</p>
+  <span style="color:red;font-weight:bold;">CAUTION ALL EMONCMS ACCOUNT DATA WILL BE OVERWRITTEN BY THE IMPORTED DATA</span><br><br>
+  <p><i>Note: Before import update to latest version of Emoncms & emonHub.</i></p>
+  <form action="<?php echo $path; ?>backup/upload" method="post" enctype="multipart/form-data">
+    <input type="file" name="file" id="file"><br><br>
+    <input class="btn btn-danger" type="submit" name="submit" value="Import Backup">
+  </form>
+  <br><br>
+  <p><i>Note: If browser upload fails for large backup files <a href="http://github.com/emoncms/backup">follow manual import instructions.</a></i></p>
+  <pre id="import-log-bound"><div id="import-log"></div></pre>
+  <br>
+  <p><i>Refresh page if log window does not update.</i></p>
+  <p><i>After import is complete logout then login using the new imported account login details.</i></p>
 </tr>
 
 </table>

diff --git a/backup/module.json b/backup/module.json
@@ -0,0 +1,4 @@
+{
+    "name"         : "Backup",
+    "version"      : "1.1.6"
+}
diff --git a/config.cfg b/config.cfg
@@ -20,8 +20,5 @@ emonhub_config_path="/home/pi/data"
 # Location of default fresh specimen emonhub config
 emonhub_specimen_config="/home/pi/emonhub/conf"
 
-# Location of emoncms.conf (leave blank if backup not required)
-emoncms_config_path="/home/pi/data"
-
 # Location of compressed backup data to import (.tar.gz)
-backup_source_path="/home/pi/data/uploads"
+backup_source_path="/home/pi/data/uploads"
diff --git a/emoncms-export.sh b/emoncms-export.sh
@@ -4,20 +4,22 @@ date=$(date +"%Y-%m-%d")
 
 echo "=== Emoncms export start ==="
 date
+echo "Backup module version:"
+cat /home/pi/backup/backup/module.json | grep version
 echo "EUID: $EUID"
 echo "Reading /home/pi/backup/config.cfg...."
 if [ -f /home/pi/backup/config.cfg ]
 then
     source /home/pi/backup/config.cfg
     echo "Location of mysql database: $mysql_path"
     echo "Location of emonhub.conf: $emonhub_config_path"
-    echo "Location of emoncms.conf: $emoncms_config_path"
     echo "Location of Emoncms: $emoncms_location"
     echo "Backup destination: $backup_location"
 else
     echo "ERROR: Backup /home/pi/backup/config.cfg file does not exist"
     exit 1
-fi  
+    sudo service feedwriter start > /dev/null
+fi
 
 #-----------------------------------------------------------------------------------------------
 # Remove Old backup files
@@ -32,7 +34,6 @@ then
     sudo rm $backup_location/emoncms-backup-$date.tar
 fi
 
-
 #-----------------------------------------------------------------------------------------------
 # Check emonPi / emonBase image version
 #-----------------------------------------------------------------------------------------------
@@ -45,17 +46,15 @@ then
     echo "Image version: $image_version"
 fi
 
-# Detect if SD card image verion, used to restore the correct emonhub.conf
-if [[ "$image_date" == "emonSD-17Jun2015" ]]
+# Very old images (the ones shipped with kickstarter campaign) have "emonpi-28May2015"
+if [[ -z $image_version ]] || [[ "$image_date" == "emonSD-17Jun2015" ]]
 then
   image="old"
 else
   image="new"
 fi
 #-----------------------------------------------------------------------------------------------
 
-
-
 sudo service feedwriter stop
 
 # Get MYSQL authentication details from settings.php
@@ -65,29 +64,52 @@ if [ -f /home/pi/backup/get_emoncms_mysql_auth.php ]; then
 else
     echo "Error: cannot read MYSQL authentication details from Emoncms settings.php"
     echo "$PWD"
+    sudo service feedwriter start > /dev/null
     exit 1
 fi
 
 # MYSQL Dump Emoncms database
-if [ -n "$username" ]; then # if username sring is not empty
+if [ -n "$username" ]; then # if username string is not empty
     mysqldump -u$username -p$password emoncms > $backup_location/emoncms.sql
+    if [ $? -ne 0 ]; then
+        echo "Error: failed to export mysql data"
+        echo "emoncms export failed"
+        sudo service feedwriter start > /dev/null
+        exit 1
+    fi
+
 else
     echo "Error: Cannot read MYSQL authentication details from Emoncms settings.php"
+    sudo service feedwriter start > /dev/null
     exit 1
 fi
 
-echo "Emoncms MYSQL database dump complete, adding files to archive .."
+echo "Emoncms MYSQL database dump complete, adding to archive..."
 
 # Create backup archive and add config files stripping out the path
-tar -cf $backup_location/emoncms-backup-$date.tar $backup_location/emoncms.sql $emonhub_config_path/emonhub.conf $emoncms_config_path/emoncms.conf $emoncms_location/settings.php /home/pi/data/node-red/flows_emonpi.json /home/pi/data/node-red/flows_emonpi_cred.json /home/pi/data/node-red/settings.js --transform 's?.*/??g'
+tar -cf $backup_location/emoncms-backup-$date.tar $backup_location/emoncms.sql $emonhub_config_path/emonhub.conf $emoncms_location/settings.php --transform 's?.*/??g' 2>&1
+if [ $? -ne 0 ]; then
+    echo "Error: failed to tar config data"
+    echo "emoncms export failed"
+    sudo service feedwriter start > /dev/null
+    exit 1
+fi
 
+echo "Adding phpfina feed data to archive..."
 # Append database folder to the archive with absolute path
-tar --append --verbose --file=$backup_location/emoncms-backup-$date.tar -C $mysql_path phpfina phptimeseries
+tar -vr --file=$backup_location/emoncms-backup-$date.tar -C $mysql_path phpfina
+echo "Adding phptimeseries feed data to archive..."
+tar -vr --file=$backup_location/emoncms-backup-$date.tar -C $mysql_path phptimeseries
 
 # Compress backup
 echo "Compressing archive..."
-gzip -f $backup_location/emoncms-backup-$date.tar
-
+gzip -fv $backup_location/emoncms-backup-$date.tar 2>&1
+if [ $? -ne 0 ]; then
+    echo "Error: failed to compress tar file"
+    echo "emoncms export failed"
+    sudo service feedwriter start > /dev/null
+    exit 1
+fi
 
 sudo service feedwriter start > /dev/null
 

diff --git a/emoncms-import.sh b/emoncms-import.sh
@@ -5,14 +5,15 @@ data_path="/home/pi/data"
 
 echo "=== Emoncms import start ==="
 date +"%Y-%m-%d-%T"
+echo "Backup module version:"
+cat /home/pi/backup/backup/module.json | grep version
 echo "EUID: $EUID"
 echo "Reading /home/pi/backup/config.cfg...."
 if [ -f /home/pi/backup/config.cfg ]
 then
     source /home/pi/backup/config.cfg
     echo "Location of mysql database: $mysql_path"
     echo "Location of emonhub.conf: $emonhub_config_path"
-    echo "Location of emoncms.conf: $emoncms_config_path"
     echo "Location of Emoncms: $emoncms_location"
     echo "Backup destination: $backup_location"
     echo "Backup source path: $backup_source_path"
@@ -80,7 +81,7 @@ if [ ! -d  $backup_location/import ]; then
 	sudo chown pi $backup_location/import -R
 fi
 
-tar xfz $backup_source_path/$backup_filename -C $backup_location/import 2>&1
+tar xfzv $backup_source_path/$backup_filename -C $backup_location/import 2>&1
 if [ $? -ne 0 ]; then
 	echo "Error: failed to decompress backup"
 	echo "$backup_source_path/$backup_filename has not been removed for diagnotics"
@@ -136,12 +137,10 @@ fi
 # cleanup
 sudo rm $backup_location/import/emoncms.sql
 
-# Save previous config settings as old.emonhub.conf and old.emoncms.conf
+# Save previous config settings as old.emonhub.conf
 echo "Import emonhub.conf > $emonhub_config_path/old.emohub.conf"
 mv $backup_location/import/emonhub.conf $emonhub_config_path/old.emonhub.conf
-echo "Import emoncms.conf > $emonhub_config_path/old.emoncms.conf"
-mv $backup_location/import/emoncms.conf $emoncms_config_path/old.emoncms.conf
-echo "conf files restored as old.*.conf, original files not modified. Please merge manually."
+echo "emonhub.conf files restored as old.emonhub.conf, original file not modified. Please merge manually."
 
 
 # Start with blank emonhub.conf
@@ -156,11 +155,6 @@ else    # Newer Feb15+ image use latest emonhub.conf with MQTT node variable top
    cp $emonhub_specimen_config/emonpi.default.emonhub.conf $emonhub_config_path/emonhub.conf
 fi
 
-# Create blank emoncms.conf and ensure permissions are correct
-sudo touch $emoncms_config_path/emoncms.conf
-sudo chown pi:www-data $emoncms_config_path/emoncms.conf
-sudo chmod 664 $emoncms_config_path/emoncms.conf
-
 redis-cli "flushall" 2>&1
 
 if [ -f /home/pi/emonpi/emoncmsdbupdate.php ]; then

diff --git a/get_emoncms_feed_datadir.php b/get_emoncms_feed_datadir.php
@@ -0,0 +1,10 @@
+<?php
+chdir("/var/www/emoncms");
+define('EMONCMS_EXEC', 1);
+require "process_settings.php";
+
+foreach ($feed_settings as $engine=>$entry) {
+if (isset($feed_settings[$engine]) && isset($feed_settings[$engine]["datadir"]))
+    echo $engine.'_location="'.$feed_settings[$engine]["datadir"].'"'."\n\n";
+}
+
diff --git a/readme.md b/readme.md
@@ -1,4 +1,3 @@
-
 ## Emoncms backup export and import tool for backup and migration
 
 * Export a compressed archive containing Emoncms Inputs, Feed data, Dashboards & config.
@@ -48,7 +47,10 @@ Follow on screen prompts, RasPi will shutdown when process is compleate. It can
     git clone https://github.com/emoncms/backup.git
     ln -s /home/pi/backup/backup/ /var/www/emoncms/Modules/backup
 
-**Note: Ensure you are running the latest version of Emoncms on the Stable branch. [A change was merged on the 9th Feb 16 to Emoncms core](https://github.com/emoncms/emoncms/commit/e83ad78e6155275d7537104367b8d44ef63d78fe) that enables symlinked modules which is essential for backup module to appear in Emoncms**
+**Note: 
+
+- Ensure you are running the latest version of Emoncms on the Stable branch. [A change was merged on the 9th Feb 16 to Emoncms core](https://github.com/emoncms/emoncms/commit/e83ad78e6155275d7537104367b8d44ef63d78fe) that enables symlinked modules which is essential for backup module to appear in Emoncms**
+- As of June 18 Backup module requires Redis to set service runner flags
 
 **If your running the older 'low-write' branch of Emoncms emonSD-17Jun15 or before then you won't be able to update to the latest version to enable symlinks, to get around this after installing the module browse to [http://emonpi/emoncms/backup](http://emonpi/emoncms/backup)**
 
@@ -72,16 +74,26 @@ or saving log in var log
 
 ## PHP Config
 
-In order to enable uploads of backup zip files we need to set the maximum upload size to be larger than the file we want to upload. This can be set system wide in `/etc/php5/apache2/php.ini`:
+In order to enable uploads of backup zip files we need to set the maximum upload size to be larger than the file we want to upload.
+
+If running php5
+
+If using php5
 
     sudo nano /etc/php5/apache2/php.ini
+
+If runnning php7 (Stretch onwards)
+
 
+    sudo nano /etc/php/7.0/apache2/php.ini
+
 Use `[CTRL + W]` to search test
 
 Set:
 
-    post_max_size = 200M
-    upload_max_filesize = 200M
+    post_max_size = 3G
+    upload_max_filesize = 3G
+    upload_tmp_dir = /home/pi/data/uploads
 
 # Create uploads folder