diff --git a/.github/workflows/spelling/expect.txt b/.github/workflows/spelling/expect.txt index f1cd2332..4711b7e3 100644 --- a/.github/workflows/spelling/expect.txt +++ b/.github/workflows/spelling/expect.txt @@ -26,6 +26,7 @@ attr austingroupbugs autocommit autoconf +autocreation autom automake automounting diff --git a/CHANGES b/CHANGES index c59adfea..916aa4b2 100644 --- a/CHANGES +++ b/CHANGES @@ -11,6 +11,7 @@ znapzend (0.21.3) unstable; urgency=medium * Added rc-script and integration documentation for FreeBSD and similar platforms * Converted configure.ac and numerous Makefile.am to avoid GNU Make syntax in favor of portability: tested with Solaris/illumos Sun make and with FreeBSD make * Extended `--autoCreation` effect (or lack thereof) to newly appearing sub-datasets; added a `--noautoCreation` option to help override configuration file settings (where used) + * Introduced `dst_N_autocreation` setting via ZFS properties (per-destination, inheritable) -- Jim Klimov Tue, 12 Mar 2024 13:42:28 +0100 diff --git a/bin/znapzend b/bin/znapzend index d8efbe3f..e4492985 100755 --- a/bin/znapzend +++ b/bin/znapzend @@ -85,6 +85,9 @@ sub main { $opts->{forbidDestRollback} = 0; } + # Note: default is "undef" to use a ZFS property dst_N_autocreation + # (lower-case "c" in the name) if present; finally assumes 0 (false) + # if not set in any configuration source for a particular dataset. if (defined($opts->{noautoCreation})) { $opts->{autoCreation} = 0; delete $opts->{noautoCreation}; diff --git a/bin/znapzendzetup b/bin/znapzendzetup index 77d376df..4470c939 100755 --- a/bin/znapzendzetup +++ b/bin/znapzendzetup @@ -402,6 +402,54 @@ sub main { last; }; + /^enable-dst-autocreation$/ && do { + $opts->{dst} = pop @ARGV; + $opts->{src} = pop @ARGV; + if (!defined $opts->{src}) { + pod2usage(-exitval => 'NOEXIT'); + die ("ERROR: source argument for option $mainOpt was not provided\n"); + } + if (!defined $opts->{dst}) { + pod2usage(-exitval => 'NOEXIT'); + die ("ERROR: destination argument for option $mainOpt was not provided\n"); + } + $zConfig->enableBackupSetDstAutoCreation($opts->{src}, $opts->{dst}) + or die "ERROR: cannot enable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n"; + + last; + }; + /^disable-dst-autocreation$/ && do { + $opts->{dst} = pop @ARGV; + $opts->{src} = pop @ARGV; + if (!defined $opts->{src}) { + pod2usage(-exitval => 'NOEXIT'); + die ("ERROR: source argument for option $mainOpt was not provided\n"); + } + if (!defined $opts->{dst}) { + pod2usage(-exitval => 'NOEXIT'); + die ("ERROR: destination argument for option $mainOpt was not provided\n"); + } + $zConfig->disableBackupSetDstAutoCreation($opts->{src}, $opts->{dst}) + or die "ERROR: cannot disable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n"; + + last; + }; + /^inherit-dst-autocreation$/ && do { + $opts->{dst} = pop @ARGV; + $opts->{src} = pop @ARGV; + if (!defined $opts->{src}) { + pod2usage(-exitval => 'NOEXIT'); + die ("ERROR: source argument for option $mainOpt was not provided\n"); + } + if (!defined $opts->{dst}) { + pod2usage(-exitval => 'NOEXIT'); + die ("ERROR: destination argument for option $mainOpt was not provided\n"); + } + $zConfig->inheritBackupSetDstAutoCreation($opts->{src}, $opts->{dst}) + or die "ERROR: cannot disable backup config for $opts->{src} destination $opts->{dst}. Did you create this config?\n"; + + last; + }; /^list$/ && do { GetOptions($opts, (@ROOT_EXEC_OPTIONS, qw(recursive|r inherited))) or exit 1; @@ -591,6 +639,12 @@ and where 'command' and its unique options is one of the following: disable-dst + enable-dst-autocreation + + disable-dst-autocreation + + inherit-dst-autocreation + list [--recursive] [--inherited] [src_dataset...] export diff --git a/lib/ZnapZend.pm b/lib/ZnapZend.pm index d3bb007c..4517e91d 100644 --- a/lib/ZnapZend.pm +++ b/lib/ZnapZend.pm @@ -49,7 +49,7 @@ has pidfile => sub { q{} }; has forcedSnapshotSuffix => sub { q{} }; has defaultPidFile => sub { q{/var/run/znapzend.pid} }; has terminate => sub { 0 }; -has autoCreation => sub { 0 }; +has autoCreation => sub { undef }; has timeWarp => sub { undef }; has nodelay => sub { 0 }; has skipOnPreSnapCmdFail => sub { 0 }; @@ -404,6 +404,15 @@ my $refreshBackupPlans = sub { #create backup hashes for all destinations for (keys %$backupSet){ my ($key) = /^dst_([^_]+)_plan$/ or next; + my $autoCreation = $self->autoCreation; + if (!defined($autoCreation)) { + # Caller did not require any particular behavior, so + # check the ZFS property name (note lower-case "c"): + $autoCreation = (exists $backupSet->{"dst_$key" . '_autocreation'} ? $backupSet->{"dst_$key" . '_autocreation'} : undef); + } + if (!defined($autoCreation)) { + $autoCreation = 0; + } #check if destination exists (i.e. is valid) otherwise recheck as dst might be online, now if (!$backupSet->{"dst_$key" . '_valid'}){ @@ -411,7 +420,7 @@ my $refreshBackupPlans = sub { $backupSet->{"dst_$key" . '_valid'} = $self->zZfs->dataSetExists($backupSet->{"dst_$key"}) or do { - if ($self->autoCreation && !$self->sendRaw) { + if ($autoCreation && !$self->sendRaw) { my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//; # check if we can access destination zpool, if so create parent dataset @@ -429,7 +438,7 @@ my $refreshBackupPlans = sub { $backupSet->{"dst_$key" . '_valid'} or $self->zLog->warn("destination '" . $backupSet->{"dst_$key"} . "' does not exist or is offline. will be rechecked every run..." - . ( $self->autoCreation ? "" : " Consider running znapzend --autoCreation" ) ); + . ( $autoCreation ? "" : " Consider running znapzend --autoCreation" ) ); }; $self->zLog->debug('refreshBackupPlans(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug); @@ -563,10 +572,23 @@ my $sendRecvCleanup = sub { #recheck non valid dst as it might be online, now if (!$backupSet->{"dst_$key" . '_valid'}) { + my $autoCreation = $self->autoCreation; + if (!defined($autoCreation)) { + # Caller did not require any particular behavior, so + # check the ZFS property name (note lower-case "c"). + # Note we are looking at "root" datasets with a backup + # schedule here (enumerated earlier); children if any + # would be checked below: + $autoCreation = (exists $backupSet->{"dst_$key" . '_autocreation'} ? $backupSet->{"dst_$key" . '_autocreation'} : undef); + } + if (!defined($autoCreation)) { + $autoCreation = 0; + } + $backupSet->{"dst_$key" . '_valid'} = $self->zZfs->dataSetExists($backupSet->{"dst_$key"}) or do { - if ($self->autoCreation && !$self->sendRaw) { + if ($autoCreation && !$self->sendRaw) { my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//; # check if we can access destination zpool, if so - @@ -584,12 +606,18 @@ my $sendRecvCleanup = sub { } }; } - ( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $self->autoCreation) ) or do { + ( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $autoCreation) ) or do { my $errmsg = "destination '" . $backupSet->{"dst_$key"} . "' does not exist or is offline; ignoring it for this round..."; - $self->zLog->warn($errmsg); - push (@sendFailed, $errmsg); - $thisSendFailed = 1; + # Avoid spamming for every loop cycle, if we do not have + # the dataset and know we do not intend to auto-create it + $self->zLog->warn($errmsg) if ($autoCreation or $self->debug); + if (!$autoCreation) { + $self->zLog->warn("Autocreation is disabled for this dataset or whole run, so skipping without error") if ($self->debug); + } else { + push (@sendFailed, $errmsg); + $thisSendFailed = 1; + } next; }; }; @@ -607,6 +635,26 @@ my $sendRecvCleanup = sub { my $dstDataSet = $srcDataSet; $dstDataSet =~ s/^\Q$backupSet->{src}\E/$backupSet->{$dst}/; + my $autoCreation = $self->autoCreation; + if (!defined($autoCreation)) { + # Caller did not require any particular behavior, so + # check the ZFS property name (note lower-case "c"). + # Look at properties of this dataset, allow inherited + # values. TOTHINK: Get properties once for all tree? + my $properties = $self->zZfs->getDataSetProperties($srcDataSet, 0, 1); + if ($properties->[0]) { + for my $prop (keys %{$properties->[0]}) { + if ($prop eq "dst_$key" . '_autocreation') { + $autoCreation = (%{$properties->[0]}{$prop} eq "on" ? 1 : 0); + last; + } + } + } + } + if (!defined($autoCreation)) { + $autoCreation = 0; + } + my $srcDataSetDisabled = (grep (/^\Q$srcDataSet\E$/, @dataSetsExplicitlyDisabled)); $self->zLog->debug('sending snapshots from ' . $srcDataSet . ' to ' . $dstDataSet . ($srcDataSetDisabled ? ": not enabled, skipped" : "")); @@ -616,14 +664,20 @@ my $sendRecvCleanup = sub { # Time to check if the target sub-dataset exists # at all (unless we would auto-create one anyway). - if (!$self->autoCreation && !$self->sendRaw && !$self->zZfs->dataSetExists($dstDataSet)) { + if ((!$autoCreation || !$self->sendRaw) && !($self->zZfs->dataSetExists($dstDataSet))) { my $errmsg = "sub-destination '" . $dstDataSet . "' does not exist or is offline; ignoring it for this round... Consider " - . ( $self->autoCreation || $self->sendRaw ? "" : "running znapzend --autoCreation or " ) + . ( $autoCreation || $self->sendRaw ? "" : "running znapzend --autoCreation or " ) . "disabling this dataset from znapzend handling."; - $self->zLog->warn($errmsg); - push (@sendFailed, $errmsg); - $thisSendFailed = 1; + # Avoid spamming for every loop cycle, if we do not have + # the dataset and know we do not intend to auto-create it + $self->zLog->warn($errmsg) if ($autoCreation or $self->debug); + if (!$autoCreation) { + $self->zLog->warn("Autocreation is disabled for this dataset or whole run, so skipping without error") if ($self->debug); + } else { + push (@sendFailed, $errmsg); + $thisSendFailed = 1; + } next; } @@ -635,7 +689,7 @@ my $sendRecvCleanup = sub { 'since=="' . $self->since . '"'. ', skipIntermediates=="' . $self->skipIntermediates . '"' . ', forbidDestRollback=="' . $self->forbidDestRollback . '"' . - ', autoCreation=="' . $self->autoCreation . '"' . + ', autoCreation=="' . ( $autoCreation ? "true" : "false" ) . '"' . ', sendRaw=="' . $self->sendRaw . '"' . ', valid=="' . ( $backupSet->{"dst_$key" . '_valid'} ? "true" : "false" ) . '"' . ', justCreated=="' . ( $backupSet->{"dst_$key" . '_justCreated'} ? "true" : "false" ) . '"' diff --git a/lib/ZnapZend/Config.pm b/lib/ZnapZend/Config.pm index 4b5e80b0..19b9dc6f 100644 --- a/lib/ZnapZend/Config.pm +++ b/lib/ZnapZend/Config.pm @@ -86,7 +86,12 @@ my $checkBackupSets = sub { my $self = shift; for my $backupSet (@{$self->backupSets}){ - + # Note that we only normally call this either when we walk all + # known backup/retention schedules (datasets with at least one + # local "org.znapzend:..." property), or just once for a single + # "--runonce=..." backupSet (not recursing into children with + # their exceptional settings then, unless also "--recursive"). + # # In case there is only one property on this dataset, which is the # "enabled" flag and is set to "off"; consider it a normal situation # and do not even notify it. This situation will appear when there @@ -94,15 +99,35 @@ my $checkBackupSets = sub { # Note: backupSets will have at least the key "src". Therefore, we # need to skip the dataset if there are two properties and one of # them is "enabled". - if (keys(%{$backupSet}) eq 2 && exists($backupSet->{"enabled"})){ - next; - } - + # # Similarly for datasets which declare both the "enabled" flag and # the "recursion" flag (e.g. to prune whole dataset sub-trees from # backing up with znapzend) by configuring only the root of such # sub-tree. - if (keys(%{$backupSet}) eq 3 && exists($backupSet->{"enabled"}) && exists($backupSet->{"recursive"})){ + # + # Likewise, skip checking datasets (enabled or not) that have only + # an autoCreation setting for particular destination(s); note that + # ZFS property names must be lower-case (so "c" is small here). + # Hence we prepare a filtered set of configuration keys ("dst" name + # tags are user-provided and not too predictable), so only "src" + # would remain there: + my @backupSetKeysFiltered = grep (!/^dst_[^_]+_autocreation$/, keys(%{$backupSet})); + my $backupSetKeysFiltered = scalar(@backupSetKeysFiltered); + $self->zLog->debug("#checkBackupSets# backupSetKeysFiltered " + . "for '" . $backupSet->{src} . "' = (" + . $backupSetKeysFiltered . ")[" + . join(", ", @backupSetKeysFiltered) . "]" + ) if $self->debug; + + # "src" and "enabled", or "src" alone (after disregarding autocreation): + if ( ($backupSetKeysFiltered eq 2 and exists($backupSet->{"enabled"})) + or $backupSetKeysFiltered eq 1 + ) { + next; + } + + # "src", "enabled" and "recursion" (after disregarding autocreation): + if ($backupSetKeysFiltered eq 3 && exists($backupSet->{"enabled"}) && exists($backupSet->{"recursive"})){ next; } @@ -515,6 +540,142 @@ sub disableBackupSetDst { return 0; } +sub enableBackupSetDstAutoCreation { + my $self = shift; + my $dataSet = shift; + my $dest = shift; + my $recurse = shift; # may be undef + my $inherit = shift; # may be undef + + $self->zfs->dataSetExists($dataSet) or die "ERROR: dataset $dataSet does not exist\n"; + + $self->backupSets($self->zfs->getDataSetProperties($dataSet, $recurse, $inherit)); + + if (@{$self->backupSets}){ + my %cfg = %{$self->backupSets->[0]}; + + if ( !($dest =~ /^dst_[^_]+$/) ) { + if ($cfg{'dst_' . $dest}) { + # User passed valid key of the destination config, + # convert to zfs attribute/perl struct name part + $dest = 'dst_' . $dest; + } elsif ($dest =~ /^DST:/) { + my $desttemp = $dest; + $desttemp =~ s/^DST:// ; + if ($cfg{'dst_' . $desttemp}) { + # User passed valid key of the destination config, + # convert to zfs attribute/perl struct name part + $dest = 'dst_' . $desttemp; + } + } + # TODO: Else search by value of 'dst_N' as a "(remote@)dataset" + } + + if ($cfg{$dest}) { + if ($cfg{$dest . '_autocreation'}) { + $cfg{$dest . '_autocreation'} = 'on'; + } + } else { + die "ERROR: dataset $dataSet backup plan does not have destination $dest\n"; + } + $self->setBackupSet(\%cfg); + + return 1; + } + + return 0; +} + +sub disableBackupSetDstAutoCreation { + my $self = shift; + my $dataSet = shift; + my $dest = shift; + my $recurse = shift; # may be undef + my $inherit = shift; # may be undef + + $self->zfs->dataSetExists($dataSet) or die "ERROR: dataset $dataSet does not exist\n"; + + $self->backupSets($self->zfs->getDataSetProperties($dataSet, $recurse, $inherit)); + + if (@{$self->backupSets}){ + my %cfg = %{$self->backupSets->[0]}; + + if ( !($dest =~ /^dst_[^_]+$/) ) { + if ($cfg{'dst_' . $dest}) { + # User passed valid key of the destination config, + # convert to zfs attribute/perl struct name part + $dest = 'dst_' . $dest; + } elsif ($dest =~ /^DST:/) { + my $desttemp = $dest; + $desttemp =~ s/^DST:// ; + if ($cfg{'dst_' . $desttemp}) { + # User passed valid key of the destination config, + # convert to zfs attribute/perl struct name part + $dest = 'dst_' . $desttemp; + } + } + # TODO: Else search by value of 'dst_N' as a "(remote@)dataset" + } + + if ($cfg{$dest}) { + $cfg{$dest . '_autocreation'} = 'off'; + } else { + die "ERROR: dataset $dataSet backup plan does not have destination $dest\n"; + } + $self->setBackupSet(\%cfg); + + return 1; + } + + return 0; +} + +sub inheritBackupSetDstAutoCreation { + my $self = shift; + my $dataSet = shift; + my $dest = shift; + my $recurse = shift; # may be undef + my $inherit = shift; # may be undef + + $self->zfs->dataSetExists($dataSet) or die "ERROR: dataset $dataSet does not exist\n"; + + $self->backupSets($self->zfs->getDataSetProperties($dataSet, $recurse, $inherit)); + + if (@{$self->backupSets}){ + my %cfg = %{$self->backupSets->[0]}; + + if ( !($dest =~ /^dst_[^_]+$/) ) { + if ($cfg{'dst_' . $dest}) { + # User passed valid key of the destination config, + # convert to zfs attribute/perl struct name part + $dest = 'dst_' . $dest; + } elsif ($dest =~ /^DST:/) { + my $desttemp = $dest; + $desttemp =~ s/^DST:// ; + if ($cfg{'dst_' . $desttemp}) { + # User passed valid key of the destination config, + # convert to zfs attribute/perl struct name part + $dest = 'dst_' . $desttemp; + } + } + # TODO: Else search by value of 'dst_N' as a "(remote@)dataset" + } + + if ($cfg{$dest}) { + if ($cfg{$dest . '_autocreation'}) { + $cfg{$dest . '_autocreation'} = undef; + } + } else { + die "ERROR: dataset $dataSet backup plan does not have destination $dest\n"; + } + $self->setBackupSet(\%cfg); + + return 1; + } + + return 0; +} + 1; __END__