Extend --autoCreation effect (or default lack thereof) to newly appearing sub-datasets

.github/workflows/spelling/expect.txt

@@ -379,6 +379,7 @@ newestbe
 nf
 nh
 noaction
+noauto
 nobase
 nodelay
 nodestroy

CHANGES

@@ -10,6 +10,7 @@ znapzend (0.21.3) unstable; urgency=medium
   * Fixed CI recipes and contents for spell-checker
   * Added rc-script and integration documentation for FreeBSD and similar platforms
   * Converted configure.ac and numerous Makefile.am to avoid GNU Make syntax in favor of portability: tested with Solaris/illumos Sun make and with FreeBSD make
+  * Extended `--autoCreation` effect (or lack thereof) to newly appearing sub-datasets; added a `--noautoCreation` option to help override configuration file settings (where used)
 
  -- Jim Klimov <[email protected]>  Tue, 12 Mar 2024 13:42:28 +0100
 

bin/znapzend

@@ -16,7 +16,7 @@
     my $opts = {};
     GetOptions($opts, qw(help|h man debug|d noaction|n nodestroy features=s),
         qw(sudo pfexec rootExec=s daemonize pidfile=s logto=s loglevel=s),
-        qw(runonce:s connectTimeout=s timeWarp=i nodelay autoCreation version),
+        qw(runonce:s connectTimeout=s timeWarp=i nodelay autoCreation noautoCreation version),
         qw(forcedSnapshotSuffix=s forbidDestRollback),
         qw(skipIntermediates|i sendIntermediates|I),
             # Note: the intended usage is either via feature as done
@@ -48,7 +48,7 @@
 
     if (defined($opts->{sendIntermediates})) {
         if ( (defined($opts->{skipIntermediates})) && ($opts->{skipIntermediates} != 0) ) {
             warn "Options skipIntermediates and sendIntermediates are exclusive; and sendIntermediates wins!";
         }
         # Note that the legacy default is to sendIntermediates.
         # Beware or benefit that this can deliver manually named
@@ -85,6 +85,15 @@
         $opts->{forbidDestRollback} = 0;
     }
 
+    if (defined($opts->{noautoCreation})) {
+        $opts->{autoCreation} = 0;
+        delete $opts->{noautoCreation};
+    } else {
+        if (defined($opts->{autoCreation})) {
+            $opts->{autoCreation} = 1;
+        }
+    }
+
     if (defined($opts->{sinceForced}) && ($opts->{sinceForced} eq '')) {
         delete $opts->{sinceForced};
     }
@@ -102,7 +111,7 @@
     } else {
         if (defined($opts->{since})) {
             $opts->{forbidDestRollback} = 1;
             warn "The --since option is used and will try to ensure that the snapshot exists in history of destinations only if an older snapshot is the latest common (would not delete and rewrite subsequent snapshots to make way)"
         }
     }
 
@@ -202,6 +211,8 @@
  --connectTimeout=x     sets the ConnectTimeout for ssh commands
  --autoCreation         automatically create dataset on destination if it does
                         not exist
+ --noautoCreation       avoid automatically creating a dataset on destination
+                        if it does not exist (default)
  --timeWarp=x           shift znapzend's sense of NOW into the future
                         by x seconds
  --skipOnPreSnapCmdFail skip snapshots if the pre-snap-command fails
@@ -610,6 +621,12 @@
 
 Automatically create a dataset on a destination host if it's not there yet.
 
+=item B<--noautoCreation>
+
+Avoid automatically creating a dataset on a destination host if it's not
+there yet. This is the default behavior; the option is available to help
+explicitly override a setting inherited from a configuration file, etc.
+
 =item B<--timeWarp>=x
 
 Shift ZnapZend's sense of time into the future by x seconds.

lib/ZnapZend.pm

@@ -431,6 +431,8 @@ my $refreshBackupPlans = sub {
                             . "' does not exist or is offline. will be rechecked every run..."
                             . ( $self->autoCreation ? "" : " Consider running znapzend --autoCreation" ) );
                 };
+
+                $self->zLog->debug('refreshBackupPlans(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug);
             }
             $backupSet->{"dst$key" . 'PlanHash'}
                 = $self->zTime->backupPlanToHash($backupSet->{"dst_$key" . '_plan'});
@@ -567,7 +569,10 @@ my $sendRecvCleanup = sub {
                 if ($self->autoCreation && !$self->sendRaw) {
                     my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//;
 
-                    # check if we can access destination zpool, if so create parent dataset
+                    # check if we can access destination zpool, if so -
+                    # create parent dataset (e.g. this backupSet root;
+                    # note that if we recurse into children that may be
+                    # absent, they are treated separately below)
                     $self->zZfs->dataSetExists($zpool) && do {
                         $self->zLog->info("creating destination dataset '" . $backupSet->{"dst_$key"} . "'...");
 
@@ -581,13 +586,15 @@ my $sendRecvCleanup = sub {
                 }
                 ( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $self->autoCreation) ) or do {
                     my $errmsg = "destination '" . $backupSet->{"dst_$key"}
-                        . "' does not exist or is offline. ignoring it for this round...";
+                        . "' does not exist or is offline; ignoring it for this round...";
                     $self->zLog->warn($errmsg);
                     push (@sendFailed, $errmsg);
                     $thisSendFailed = 1;
                     next;
                 };
             };
+
+            $self->zLog->debug('sendRecvCleanup(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug);
         }
 
         #sending loop through all subdatasets
@@ -607,6 +614,19 @@ my $sendRecvCleanup = sub {
                 next;
             }
 
+            # Time to check if the target sub-dataset exists
+            # at all (unless we would auto-create one anyway).
+            if (!$self->autoCreation && !$self->sendRaw && !$self->zZfs->dataSetExists($dstDataSet)) {
+                my $errmsg = "sub-destination '" . $dstDataSet
+                    . "' does not exist or is offline; ignoring it for this round... Consider "
+                    . ( $self->autoCreation || $self->sendRaw ? "" : "running znapzend --autoCreation or " )
+                    . "disabling this dataset from znapzend handling.";
+                $self->zLog->warn($errmsg);
+                push (@sendFailed, $errmsg);
+                $thisSendFailed = 1;
+                next;
+            }
+
             {   # scoping
                 local $@;
                 eval {
@@ -615,8 +635,12 @@ my $sendRecvCleanup = sub {
                         'since=="' . $self->since . '"'.
                         ', skipIntermediates=="' . $self->skipIntermediates . '"' .
                         ', forbidDestRollback=="' . $self->forbidDestRollback . '"' .
+                        ', autoCreation=="' . $self->autoCreation . '"' .
+                        ', sendRaw=="' . $self->sendRaw . '"' .
+                        ', valid=="' . ( $backupSet->{"dst_$key" . '_valid'} ? "true" : "false" ) . '"' .
                         ', justCreated=="' . ( $backupSet->{"dst_$key" . '_justCreated'} ? "true" : "false" ) . '"'
                         ) if $self->debug;
+
                     if ($self->since) {
                         # Make sure that if we use the "--sinceForced=X" or
                         # "--since=X" option, this named snapshot exists (or

lib/ZnapZend/Config.pm

@@ -204,6 +204,7 @@ my $checkBackupSets = sub {
         for my $dst (grep { /^dst_[^_]+$/ } keys %$backupSet){
             #store backup destination validity. will be checked where used
             $backupSet->{$dst . '_valid'} = $self->zfs->dataSetExists($backupSet->{$dst});
+            $self->zLog->debug('checkBackupSets(): detected ' . $dst . '_valid status for ' . $backupSet->{$dst} . ': ' . $backupSet->{$dst . '_valid'}) if ($self->debug);
 
             #if a backup destination is given, we also need a plan
             $backupSet->{$dst . '_plan'} or die "ERROR: no backup plan given for destination\n";

man/znapzend.1

@@ -176,6 +176,8 @@ znapzend \- znapzend daemon
 \& \-\-connectTimeout=x     sets the ConnectTimeout for ssh commands
 \& \-\-autoCreation         automatically create dataset on destination if it does
 \&                        not exist
+\& \-\-noautoCreation       avoid automatically creating a dataset on destination
+\&                        if it does not exist (default)
 \& \-\-timeWarp=x           shift znapzend\*(Aqs sense of NOW into the future
 \&                        by x seconds
 \& \-\-skipOnPreSnapCmdFail skip snapshots if the pre\-snap\-command fails
@@ -569,6 +571,11 @@ sets the ssh connection timeout (in seconds)
 .IP "\fB\-\-autoCreation\fR" 4
 .IX Item "--autoCreation"
 Automatically create a dataset on a destination host if it's not there yet.
+.IP "\fB\-\-noautoCreation\fR" 4
+.IX Item "--noautoCreation"
+Avoid automatically creating a dataset on a destination host if it's not
+there yet. This is the default behavior; the option is available to help
+explicitly override a setting inherited from a configuration file, etc.
 .IP "\fB\-\-timeWarp\fR=x" 4
 .IX Item "--timeWarp=x"
 Shift ZnapZend's sense of time into the future by x seconds.