From 6848f49b049478fe90665cbbb4b65d7d9bd62152 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 07:11:45 +0800 Subject: [PATCH 1/8] feat(schedules): Replicate Every Dataset copies each dataset under one dataset here Settings gains Replicate Every Dataset: each hour, or on the days ticked, every dataset is copied under Copy Under, named for its dataset, with Snapshots To Keep for each. It leaves out the pool it copies into, the datasets snapshot schedules leave out, and a dataset whose own copy schedule makes that copy. Parents go first; a missing parent becomes an empty, unmounted, read-only holder, and a dataset whose parent's copy failed waits for the next run. One notification a run; diagnostics show its row and count. --- .coderabbit.yaml | 1 + README.md | 2 +- tests/admin_route_test.php | 10 +- tests/diagnostics_bundle_test.php | 13 +- tests/diagnostics_test.php | 25 ++- tests/fixtures/bin/zfs | 11 ++ tests/run_http_inventory_tests.sh | 3 +- tests/schedule_copies_test.php | 191 ++++++++++++++++++++ tests/schedule_snapshot_test.php | 12 +- tests/settings_send_test.cjs | 29 ++- zfs.master/ZFSMasterSettings.page | 73 ++++++-- zfs.master/backend/ZFSMAdmin.php | 6 + zfs.master/include/ZFSMDiagNames.php | 7 +- zfs.master/include/ZFSMDiagPlugin.php | 14 +- zfs.master/include/ZFSMKeep.php | 4 +- zfs.master/include/ZFSMReplicaPlan.php | 1 + zfs.master/include/ZFSMReplicaTarget.php | 17 ++ zfs.master/include/ZFSMReplicate.php | 10 +- zfs.master/include/ZFSMSchedule.php | 36 ++-- zfs.master/include/ZFSMScheduleCopies.php | 137 ++++++++++++++ zfs.master/include/ZFSMScheduleSnapshot.php | 72 +++++--- 21 files changed, 580 insertions(+), 94 deletions(-) create mode 100644 tests/schedule_copies_test.php create mode 100644 zfs.master/include/ZFSMScheduleCopies.php diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 72d16c66..ebc78cab 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -39,6 +39,7 @@ reviews: - Escape anything echoed into HTML or inline script. Config is saved to `/boot`, so saved values from older versions must keep working. - File names, ZFS user properties and anything from another server may not be valid UTF-8. htmlspecialchars() needs ENT_SUBSTITUTE, or it returns '' and the text disappears. json_encode() needs JSON_INVALID_UTF8_SUBSTITUTE, or its false result must be handled; never echo, publish or hash a json_encode() that can fail. - Every `zfs receive` takes `-s -u -o readonly=on` and never `-F`. A copy of a volume must not appear as a block device (volmode=none). + - Replicate Every Dataset (include/ZFSMScheduleCopies.php) makes an empty holder (`zfs create -o canmount=off -o readonly=on -o zfs.master:holder=on`) only where a copy's parent is missing. The property inherits, so a dataset is a holder only where its source is `local` (zfsmReplicaHolder). - path: "zfs.master/{include/ZFSMDiag*.php,scripts/diagnostics.php}" instructions: | Diagnostics: a zip the owner may post in a public issue. Its privacy scope is fixed; report gaps inside it and nothing past it. diff --git a/README.md b/README.md index becceb4c..7d8aef03 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ See and manage your ZFS pools, datasets and snapshots from Unraid's **Main** tab - **Snapshots** — take, roll back, rename, clone, hold, release and delete; an icon warns when the newest gets old - **Snapshots Admin** — filter, sort, browse a snapshot's files, or see what changed since it - **Snapshot schedules** — snapshot every dataset hourly or on chosen days and keep the newest, plus the newest of each of the last days, weeks, months or years you choose; a dataset can have its own (not a read-only, legacy-mounted or Sanoid/Syncoid one), or none -- **Replication** — copy datasets to another pool or ZFS Master NG server, once or on a schedule +- **Replication** — copy datasets to another pool or ZFS Master NG server, once or on a schedule, or every dataset into one dataset here on a schedule (Replicate Every Dataset) - **Reviews** — destroy, rollback and bulk delete show what they'll remove first - **Encryption** — create encrypted datasets, lock and unlock them - **Directories** — list, move or delete folders, or convert one to a dataset (cloned instead of copied where it can) diff --git a/tests/admin_route_test.php b/tests/admin_route_test.php index 495b73a8..80cc55e0 100644 --- a/tests/admin_route_test.php +++ b/tests/admin_route_test.php @@ -30,10 +30,12 @@ 'schedulesnapshotstiers' => array('cmd' => 'schedulesnapshots', 'every' => 'hourly', 'time' => '', 'days' => '', 'keep' => ' 24 7D, 4w '), 'schedulesnapshotswrong' => array('cmd' => 'schedulesnapshots', 'every' => 'days', 'time' => '', 'days' => '9', 'keep' => 'x5'), 'schedulesnapshotsown' => array('cmd' => 'schedulesnapshots', 'zdataset' => 'tank/data', 'every' => 'days', 'time' => '04:00', 'days' => '1', 'keep' => ''), + 'schedulecopies' => array('cmd' => 'schedulecopies', 'under' => 'backup', 'every' => 'hourly', 'time' => '', 'days' => '', 'keep' => '3'), + 'schedulecopiesnowhere' => array('cmd' => 'schedulecopies', 'under' => 'nowhere', 'every' => 'hourly', 'time' => '', 'days' => '', 'keep' => ''), 'previewschedules' => array('cmd' => 'previewschedules', 'zdataset' => 'tank/data'), 'previewschedulesbad' => array('cmd' => 'previewschedules', 'zdataset' => 'tank/bad name!'), ); -$replication = array('previewreplicate', 'replicatedataset', 'schedulepicked', 'schedulenowhere', 'schedulekeep', 'schedulesnapshots', 'schedulesnapshotstiers', 'schedulesnapshotswrong', 'schedulesnapshotsown', 'previewschedules', 'previewschedulesbad'); +$replication = array('previewreplicate', 'replicatedataset', 'schedulepicked', 'schedulenowhere', 'schedulekeep', 'schedulesnapshots', 'schedulesnapshotstiers', 'schedulesnapshotswrong', 'schedulesnapshotsown', 'schedulecopies', 'schedulecopiesnowhere', 'previewschedules', 'previewschedulesbad'); $route = $argv[1] ?? ''; if (!isset($routes[$route])) exit(2); @@ -59,7 +61,7 @@ // A schedule to remove, or a snapshot schedule saved (the default or a dataset's own), on the tmpfs the runner mounts on /boot/config. No update_cron here, so // Unraid's crontab is not rebuilt. -if ($route === 'scheduleremovesaved' || $route === 'schedulesnapshots' || $route === 'schedulesnapshotstiers' || $route === 'schedulesnapshotsown') { +if ($route === 'scheduleremovesaved' || $route === 'schedulesnapshots' || $route === 'schedulesnapshotstiers' || $route === 'schedulesnapshotsown' || $route === 'schedulecopies') { zfsmTestScratch('/boot/config'); putenv('ZFSM_TEST_JOB_DIR='.sys_get_temp_dir().'/zfsm-route-jobs-'.getmypid()); @mkdir('/boot/config/plugins/zfs.master', 0700, true); @@ -114,6 +116,10 @@ 'schedulesnapshotsown' => ($answer['said'] ?? '') === 'Saved: a snapshot of each of 1 dataset(s) (tank/data and below) every Monday at 04:00; it keeps all it takes' && strpos($answer['schedules_html'] ?? '', '>Snapshots of tank/data and below, every Monday at 04:00, keeps all it takes preg_match('/^Saved: a copy of each of 1 dataset\(s\) under backup every hour at :[0-5][0-9]; it keeps the newest 3 of the snapshots of each dataset it copies$/D', $answer['said'] ?? '') === 1 + && strpos($answer['schedules_html'] ?? '', '>Replicate Every Dataset under backup, every hour at :') !== false + && array_column(json_decode(file_get_contents('/boot/config/plugins/zfs.master/schedules.json'), true), 'dest') === array('backup/'), + 'schedulecopiesnowhere' => $answer === array('error' => 'Nothing was saved: nowhere does not exist on this server') && !is_file('/boot/config/plugins/zfs.master/schedules.json'), 'previewschedules' => $answer === array('every' => 'default', 'time' => '00:00', 'days' => range(0, 6), 'keep' => '', 'inherited' => 'Snapshot Every Dataset, which is off: no scheduled snapshots', 'skipped' => ''), 'previewschedulesbad' => ($answer['failed'] ?? null) === array('request' => 'Invalid dataset name'), )[$route]; diff --git a/tests/diagnostics_bundle_test.php b/tests/diagnostics_bundle_test.php index 85671925..6a83397e 100644 --- a/tests/diagnostics_bundle_test.php +++ b/tests/diagnostics_bundle_test.php @@ -130,6 +130,10 @@ function has($needle, $text, $message) { 'Schedule 0a1b2c3e: Scheduled snapshots of tank/CanaryOwnSan not taken: tank/CanaryOwnSan has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool; remove its schedule in Settings', 'Schedule 0a1b2c3e: Scheduled snapshots of 1 of 1 dataset(s) (tank/CanaryOwnSelf and below) taken. Not taken: tank/CanaryOwnSelf has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool', 'Schedule 0a1b2c3e: Scheduled snapshots of tank/CanaryOwnOff not taken: its pool is not imported', + // Replicate Every Dataset: how many it copied, those waiting for the copy above them, a holder it could not make, one in a copy's place. + 'Schedule 0a1b2c40: Scheduled copies under tank/CanaryUnder: 1 of 8 dataset(s) copied. Not copied, as the copy of a dataset above it is missing (tank/CanaryWaitA, tank/CanaryWait B, tank/CanaryWaitC, and 2 more). ' + ."The dataset to hold the copies below tank/CanaryUnder/tank/CanaryHold could not be made: cannot create 'tank/CanaryUnder/tank/CanaryHold': out of space", + 'Replication of tank/CanaryCpSrc to tank/CanaryUnder/tank/CanaryCpSrc not started: tank/CanaryUnder/tank/CanaryCpSrc only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them', 'ZFS Dataset Rename: Snapshot schedules of tank/CanaryMvFrom failed: not moved to tank/CanaryMvTo (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules', // A schedule's Keep: what it removed, what stayed and why, and holds a run could not put on or take off. 'Removed tank/CanaryDs One@CanaryKeepOld (not among the newest 3, 7 daily and 4 weekly its schedule keeps)', @@ -306,6 +310,13 @@ function has($needle, $text, $message) { has($own.'of 1 of 1 dataset(s) (tank/'.$token('CanaryOwnSelf').' and below) taken. Not taken: tank/'.$token('CanaryOwnSelf')." has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool\n", $log, 'and one that took those below but not its own'); has($own.'of tank/'.$token('CanaryOwnOff')." not taken: its pool is not imported\n", $log, 'of one whose pool is not imported'); +$under = 'tank/'.$token('CanaryUnder'); +// Below Copy Under a copy's path names its pool again, as a dataset there: a token too. +$cp = $under.'/'.zfsmDiagToken('tank').'/'.$token('CanaryCpSrc'); +$hold = $under.'/'.zfsmDiagToken('tank').'/'.$token('CanaryHold'); +has('Schedule '.zfsmDiagToken('0a1b2c40').": Scheduled copies under $under: 1 of 8 dataset(s) copied. Not copied, as the copy of a dataset above it is missing (tank/".$token('CanaryWaitA').', tank/'.$token('CanaryWait B').', tank/'.$token('CanaryWaitC').', and 2 more). ' + ."The dataset to hold the copies below $hold could not be made: cannot create ?$hold?: out of space\n", $log, 'a run of copies under a dataset: each name a token, the plugin\'s words in clear'); +has('Replication of tank/'.$token('CanaryCpSrc')." to $cp not started: $cp only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them\n", $log, 'and one in a copy\'s place'); has('ZFS Dataset Rename: Snapshot schedules of tank/'.$token('CanaryMvFrom').' failed: not moved to tank/'.$token('CanaryMvTo').' (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules'."\n", $log, 'a rename whose schedules were not moved'); // Keep Replicas on a copy received here, which the next receive tries again. @@ -438,7 +449,7 @@ function has($needle, $text, $message) { $plugins_own = '/boot/config/plugins/ /boot/config/plugins/zfs.master/zfs.master-2026.11.11.3-x86_64-1.txz /var/log/syslog.2 /var/run/zfs.master.rewrite/123.lock /tmp/zfs.master-diag.0123456789abcdef/plugin/log.txt /boot/logs' .' /var/log/packages/zfs.master-2026.11.11.3-x86_64-1 /var/run/zfs.master.alerts.json.lock /var/run/zfs.master.diagnostics/pack.lock' .' /boot/config/plugins/zfs.master/ssh/remotes.json /boot/config/plugins/zfs.master/ssh/new-0123abcd.pub /var/run/zfs.master.replica-send/settings.lock /var/run/zfs.master.replica-send/ssh-0123456789ab/known_hosts' - .' /var/run/zfs.master.replica-send/ssh-0123456789ab/cm /boot/config/plugins/zfs.master/schedules.json /boot/config/plugins/zfs.master/schedules.cron /var/run/zfs.master.schedule/0a1b2c3d.json /var/run/zfs.master.schedule/settings.lock'; + .' /var/run/zfs.master.replica-send/ssh-0123456789ab/cm /boot/config/plugins/zfs.master/schedules.json /boot/config/plugins/zfs.master/schedules.cron /var/run/zfs.master.schedule/0a1b2c3d.json /var/run/zfs.master.schedule/0a1b2c3d.lock /var/run/zfs.master.schedule/settings.lock'; same(explode(' ', $plugins_own), array_map('zfsmDiagAnonText', explode(' ', $plugins_own)), 'the plugin\'s own paths stay, each a field of its own'); // In a row a field is the whole path: one of the plugin's names with a space and more after it is a folder's, not the plugin's. same('/var/'.zfsmDiagToken('run').'/'.zfsmDiagToken('zfs.master.rewrite ClientA')."\tlocal", zfsmDiagAnonText("/var/run/zfs.master.rewrite ClientA\tlocal"), 'a field that only starts with one of the plugin\'s names is a folder\'s'); diff --git a/tests/diagnostics_test.php b/tests/diagnostics_test.php index e1280c40..0b1c7d3e 100644 --- a/tests/diagnostics_test.php +++ b/tests/diagnostics_test.php @@ -427,19 +427,19 @@ function has($needle, $text, $message) { putenv('ZFSM_TEST_REPL_DIR='.$work.'/pool'); list($findings, $text) = $found('zfsmDiagReplicaSchedules', $snapped); same(array(array(), false), array($findings, $GLOBALS['zfsm_diag']['unread']), 'Snapshot Every Dataset with its lines in both has no findings'); -has("records in schedules.json the plugin does not use: 0\ndatasets each snapshot schedule takes now: 0b0b0b0b 2\n\n", $text, 'how many datasets it takes, read-only ones left out'); +has("records in schedules.json the plugin does not use: 0\ndatasets each schedule of snapshots or of copies under a dataset covers now: 0b0b0b0b 2\n\n", $text, 'how many datasets it takes, read-only ones left out'); has("\n0b0b0b0b\t(every dataset)\t(snapshots only)\tevery hour at :30\t2\t0\n", $text, 'its row'); putenv('ZFSM_TEST_COVER_FAIL=1'); $text = $found('zfsmDiagReplicaSchedules', $snapped)[1]; putenv('ZFSM_TEST_COVER_FAIL'); same(true, $GLOBALS['zfsm_diag']['unread'], 'a count that can\'t be read marks the file (zfsmDiagRead)'); -has("datasets each snapshot schedule takes now: (could not be read: cannot iterate filesystems: Input/output error)\n", $text, 'and says why'); +has("datasets each schedule of snapshots or of copies under a dataset covers now: (could not be read: cannot iterate filesystems: Input/output error)\n", $text, 'and says why'); putenv('ZFSM_TEST_COVER_FAIL=tank/data'); $text = $found('zfsmDiagReplicaSchedules', $snapped)[1]; putenv('ZFSM_TEST_COVER_FAIL'); -has("datasets each snapshot schedule takes now: 0b0b0b0b 1 (not every dataset could be read: cannot open 'tank/data': Input/output error)\n", $text, 'one it can\'t read: the rest counted, and it named'); +has("datasets each schedule of snapshots or of copies under a dataset covers now: 0b0b0b0b 1 (not every dataset could be read: cannot open 'tank/data': Input/output error)\n", $text, 'one it can\'t read: the rest counted, and it named'); file_put_contents($work.'/pool/marks', "tank/data@autosnap_2026-11-17_00:00:01_hourly\t1\t1\n"); -has("datasets each snapshot schedule takes now: 0b0b0b0b 1\n", $found('zfsmDiagReplicaSchedules', $snapped)[1], 'one with Sanoid\'s snapshots left out'); +has("datasets each schedule of snapshots or of copies under a dataset covers now: 0b0b0b0b 1\n", $found('zfsmDiagReplicaSchedules', $snapped)[1], 'one with Sanoid\'s snapshots left out'); file_put_contents($work.'/pool/marks', ''); // A dataset's own (Main, Schedules) and a None: each row named, each count apart; None has no cron line to miss, and is not counted. $own = array('id' => '0c0c0c0c', 'dataset' => 'tank/data', 'minute' => 45) + $every; @@ -450,8 +450,19 @@ function has($needle, $text, $message) { file_put_contents($root.'/etc/cron.d/root', zfsmScheduleCronText(array($every, $own, $none)), FILE_APPEND); }); same(array(), $findings, 'own schedules and a None with their lines have no findings'); -has("datasets each snapshot schedule takes now: 0b0b0b0b 0, 0c0c0c0c 1\n", $text, 'each counts only the datasets that follow it'); +has("datasets each schedule of snapshots or of copies under a dataset covers now: 0b0b0b0b 0, 0c0c0c0c 1\n", $text, 'each counts only the datasets that follow it'); same(1, preg_match('~^0c0c0c0c\t(name-[0-9a-f]{8})/name-[0-9a-f]{8}\t\(snapshots only\)\tevery hour at :45\t2\t0\n0d0d0d0d\t\1\t\(snapshots only\)\tnever\t0\t0$~m', $text), 'their rows, the datasets as tokens'); +// Replicate Every Dataset and a dataset's None of copies: rows named, where the copies go; counted apart from the snapshot schedule's. +$copies = array('id' => '0e0e0e0e', 'dest' => 'backup/', 'minute' => 50, 'keep' => 0) + $every; +$nocopy = array('id' => '0f0f0f0f', 'dataset' => 'tank/data', 'dest' => '/', 'every' => 'none', 'minute' => 0, 'keep' => 0) + $every; +list($findings, $text) = $found('zfsmDiagReplicaSchedules', function($root) use ($every, $copies, $nocopy) { + file_put_contents($root.'/boot/config/plugins/zfs.master/schedules.json', json_encode(array($every, $copies, $nocopy))); + file_put_contents($root.'/boot/config/plugins/zfs.master/schedules.cron', zfsmScheduleCronText(array($every, $copies, $nocopy))); + file_put_contents($root.'/etc/cron.d/root', zfsmScheduleCronText(array($every, $copies, $nocopy)), FILE_APPEND); +}); +same(array(), $findings, 'Replicate Every Dataset and a None with their lines have no findings'); +has("datasets each schedule of snapshots or of copies under a dataset covers now: 0b0b0b0b 2, 0e0e0e0e 1\n", $text, 'each counts its own: the copies leave out the dataset with None'); +same(1, preg_match('~^0e0e0e0e\t\(every dataset\)\tname-[0-9a-f]{8}/ \(each dataset under it\)\tevery hour at :50\t0\t0\n0f0f0f0f\tname-[0-9a-f]{8}/name-[0-9a-f]{8}\t\(no copies\)\tnever\t0\t0$~m', $text), 'their rows, where the copies go a token'); // A dataset only the default takes can't be read: a dataset's own after it, which never takes it, doesn't hide that. putenv('ZFSM_TEST_COVER_FAIL=backup/copy'); $text = $found('zfsmDiagReplicaSchedules', function($root) use ($every, $own) { @@ -460,7 +471,7 @@ function has($needle, $text, $message) { file_put_contents($root.'/etc/cron.d/root', zfsmScheduleCronText(array($every, $own)), FILE_APPEND); })[1]; putenv('ZFSM_TEST_COVER_FAIL'); -has("datasets each snapshot schedule takes now: 0b0b0b0b 1, 0c0c0c0c 1 (not every dataset could be read: cannot open 'backup/copy': Input/output error)\n", $text, 'the default\'s unread dataset, with an own one after it'); +has("datasets each schedule of snapshots or of copies under a dataset covers now: 0b0b0b0b 1, 0c0c0c0c 1 (not every dataset could be read: cannot open 'backup/copy': Input/output error)\n", $text, 'the default\'s unread dataset, with an own one after it'); // None never runs to clear a count a run left as it was saved: not found, its row 0; the own one's count still shows. list($findings, $text) = $found('zfsmDiagReplicaSchedules', function($root) use ($every, $own, $none, $work) { file_put_contents($root.'/boot/config/plugins/zfs.master/schedules.json', json_encode(array($every, $own, $none))); @@ -578,7 +589,7 @@ public function url_stat($path, $flags) { return self::real(function() use ($pat putenv('ZFSM_TEST_LIST_HANG'); same(array('plugin/replication-schedules.txt', 'timed out'), array_slice(explode("\t", file($work.'/hung-schedules/collectors.txt', FILE_IGNORE_NEW_LINES)[1]), 0, 2), 'the count\'s read times out'); $text = file_get_contents($work.'/hung-schedules/plugin/replication-schedules.txt'); -has("datasets each snapshot schedule takes now: (could not be read: zfs timed out)\n", $text, 'and says so'); +has("datasets each schedule of snapshots or of copies under a dataset covers now: (could not be read: zfs timed out)\n", $text, 'and says so'); has("\n0b0b0b0b\t(every dataset)\t(snapshots only)\tevery hour at :30\t2\t0\n", $text, 'with the rest of the file'); // A read its collector's limit cut short says nothing about the pool; one that ran its whole limit does. diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index 68f9ee99..50caf814 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -99,6 +99,15 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ]; then exit 0 ;; esac exists "$last_arg" || { printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1; } + # -o source: local where it was set, else inherited from the nearest dataset above that set it, as a user property is. + if [ "$5" = source ]; then + d=$last_arg + while :; do + awk -F'\t' -v n="$d" -v p="$6" '$1 == n && $2 == p { f = 1 } END { exit !f }' "$R/props" && { if [ "$d" = "$last_arg" ]; then echo local; else echo "inherited from $d"; fi; exit 0; } + [ "${d%/*}" = "$d" ] && { echo -; exit 0; } + d=${d%/*} + done + fi if [ "$5" = property,value ]; then for p in $(printf '%s' "$6" | tr , ' '); do printf '%s\t%s\n' "$p" "$(prop "$last_arg" "$p" -)"; done; exit 0; fi case "$6" in type) awk -F'\t' -v n="$last_arg" '$1 == n { print $2 }' "$R/datasets" ;; @@ -148,6 +157,8 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ]; then snap=$(printf '%s' "$header" | cut -f2); base=$(printf '%s' "$header" | cut -f3); guid=$(printf '%s' "$header" | cut -f4) [ "${ZFSM_TEST_RECV_FAIL:-0}" = changes ] && [ "$base" != - ] && [ "$base" != RESUME ] && { printf 'cannot receive incremental stream: out of space\n' >&2; exit 1; } if [ "$base" = - ] && exists "$dest" && [ "$(prop "$dest" receive_resume_token -)" = - ]; then printf "cannot receive new filesystem stream: destination '%s' exists\n" "$dest" >&2; exit 1; fi + # zfs makes a new copy only inside a dataset that exists. + exists "$dest" || exists "${dest%/*}" || { printf "cannot receive new filesystem stream: parent of '%s' does not exist\n" "$dest" >&2; exit 1; } if [ "$base" != - ] && [ "$base" != RESUME ] && [ "$(newest "$dest")" != "$(mark "$base" 2)" ]; then printf 'cannot receive incremental stream: most recent snapshot of %s does not match incremental source\n' "$dest" >&2; exit 1 fi diff --git a/tests/run_http_inventory_tests.sh b/tests/run_http_inventory_tests.sh index d4484307..2b9d025e 100644 --- a/tests/run_http_inventory_tests.sh +++ b/tests/run_http_inventory_tests.sh @@ -49,6 +49,7 @@ done "$zfsm_php" -n "$zfsm_repo/tests/replicate_test.php" "$zfsm_php" -n "$zfsm_repo/tests/schedule_test.php" "$zfsm_php" -n "$zfsm_repo/tests/schedule_snapshot_test.php" +"$zfsm_php" -n "$zfsm_repo/tests/schedule_copies_test.php" "$zfsm_php" -n "$zfsm_repo/tests/keep_test.php" # replica-serve.php fixes PATH, so the zfs it runs is the first on that PATH: this fake, only while its test runs. mount -t tmpfs tmpfs /usr/local/sbin @@ -59,6 +60,6 @@ umount /usr/local/sbin "$zfsm_php" -n "$zfsm_repo/tests/replica_remote_test.php" "$zfsm_php" -n "$zfsm_repo/tests/pool_features_test.php" sh "$zfsm_repo/tests/directory_real_fs_test.sh" "$zfsm_php" "$zfsm_repo" -for route in createdataset editdatasetproperty snapshotdataset snapshotdataset_utc rollbacksnapshot renamesnapshot destroysnapshot destroydataset previewrewrite rewritedataset previewreplicate replicatedataset schedulepicked schedulenowhere schedulekeep scheduleremove scheduleremovesaved schedulesnapshots schedulesnapshotstiers schedulesnapshotswrong schedulesnapshotsown previewschedules previewschedulesbad; do +for route in createdataset editdatasetproperty snapshotdataset snapshotdataset_utc rollbacksnapshot renamesnapshot destroysnapshot destroydataset previewrewrite rewritedataset previewreplicate replicatedataset schedulepicked schedulenowhere schedulekeep scheduleremove scheduleremovesaved schedulesnapshots schedulesnapshotstiers schedulesnapshotswrong schedulesnapshotsown schedulecopies schedulecopiesnowhere previewschedules previewschedulesbad; do "$zfsm_php" -n ${ZFSM_TEST_POSIX:+-d "extension=$ZFSM_TEST_POSIX"} "$zfsm_repo/tests/admin_route_test.php" "$route" done diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php new file mode 100644 index 00000000..443327a4 --- /dev/null +++ b/tests/schedule_copies_test.php @@ -0,0 +1,191 @@ +> ".escapeshellarg($work.'/cron.log')."\n"); + +$pool = function($file, $lines) use ($work) { file_put_contents($work.'/pool/'.$file, $lines ? implode("\n", $lines)."\n" : ''); }; +$read = function($name) use ($work) { return is_file($work.'/'.$name) ? file($work.'/'.$name, FILE_IGNORE_NEW_LINES) : array(); }; +$reset = function() use ($work) { foreach (array('notify.log', 'notify-detail.log', 'calls.log', 'cron.log') as $name) @unlink($work.'/'.$name); }; +$stored = function() { return json_decode((string)@file_get_contents(zfsmScheduleFile()), true); }; +$datasets = function() use ($read) { return array_map(function($line) { return explode("\t", $line)[0]; }, $read('pool/datasets')); }; +$snaps = function($dataset) { return array_column(zfsmKeepSnapshots($dataset) ?? array(), 'tag', 'name'); }; +$logged = function($pattern) { return preg_match($pattern.'m', (string)@file_get_contents(zfsmLogFile())); }; +$cfg = array('snap_prefix' => 'auto-', 'snap_pattern' => 'YmdHis', 'snap_utc' => 1); +$name = function($now) use ($cfg) { return zfsmNewSnapshotName($cfg, $now); }; +$tag = zfsmKeepTag('', 'backup/'); +// Three pools. Left out: the pool the copies go into, a Docker layer (legacy) and a dataset Sanoid snapshots, whose child is copied. +$fresh = function() use ($pool) { + $pool('datasets', array("tank\tfilesystem", "tank/data\tfilesystem", "tank/docker\tfilesystem", "tank/docker/0a1b\tfilesystem", "tank/san\tfilesystem", + "tank/san/kid\tfilesystem", "tank/vm\tvolume", "usb\tfilesystem", "backup\tfilesystem", "backup/old\tfilesystem")); + $pool('marks', array("tank/san@autosnap_2026-11-01_00:00:00_daily\t11\t10")); + $pool('props', array("tank/docker/0a1b\tmountpoint\tlegacy")); + $pool('holds', array()); +}; +$fresh(); +$covered = array('tank', 'tank/data', 'tank/docker', 'tank/san/kid', 'tank/vm', 'usb'); + +// The record: Replicate Every Dataset, or a dataset's own, copies under a dataset here (dest ends in /); None (dest /) only a dataset's. +$every = array('id' => '0a1b2c3d', 'dataset' => '', 'dest' => 'backup/', 'server' => '', 'every' => 'hourly', 'minute' => 30, 'hour' => 0, 'days' => range(0, 6), 'keep' => 0); +$records = array($every, array('dataset' => 'tank/data', 'every' => 'days') + $every, array('dataset' => 'tank/data', 'every' => 'none', 'dest' => '/') + $every, + array('every' => 'none', 'dest' => '/') + $every, array('dest' => '/') + $every, array('dest' => 'bad name!/') + $every, array('server' => 'nas') + $every, + array('dataset' => 'tank/data', 'every' => 'none') + $every); +same(array(true, true, true, false, false, false, false, false), array_map('zfsmScheduleValid', $records), 'which records copy under a dataset'); +// A version before this one reads any dest but '' as one copy's, whose dataset and name these never are: it runs none of them. +same(array(false, false, false), array_map('zfsmScheduleCopyValid', array_slice($records, 0, 3)), 'an older version runs none of them'); +same(array('zfsmScheduleCopiesUnder', 'zfsmScheduleSnapshotsOnly'), array(zfsmScheduleKindOf($every), zfsmScheduleKindOf(array('dest' => '') + $every)), 'each kind'); + +// It shares datasets with a snapshot schedule or a copy's that it covers, never with another of its kind. +$snapshots = array('id' => 'aaaa0000', 'dest' => '') + $every; +$copy = array('id' => 'bbbb0000', 'dataset' => 'tank/data', 'dest' => 'backup/data') + $every; +$own = function($dataset) use ($every) { return array('id' => 'cccc0000', 'dataset' => $dataset) + $every; }; +same(array(true, false, true, false, true, false, false), + array(zfsmScheduleShare($every, $snapshots), zfsmScheduleShare($every, $own('tank')), zfsmScheduleShare($own('tank'), $copy), zfsmScheduleShare($own('usb'), $copy), + zfsmScheduleShare($own('tank/data'), array('dataset' => 'tank') + $snapshots), zfsmScheduleShare($own('usb'), array('dataset' => 'tank') + $snapshots), + zfsmScheduleShare(array('every' => 'none', 'dest' => '/') + $own('tank'), $snapshots)), 'which schedules share a dataset'); + +// Covered: each dataset but those left out, by pool; not one whose own copy's schedule makes that copy. +$listed = function() use ($cfg) { return zfsmScheduleCoverRead($cfg, 10); }; +same(array(array('tank' => array('tank', 'tank/data', 'tank/docker', 'tank/san/kid', 'tank/vm'), 'usb' => array('usb')), ''), zfsmScheduleCovered($listed(), $every, array($every)), 'it covers each dataset but those left out'); +same(array('tank', 'tank/docker', 'tank/san/kid', 'tank/vm'), zfsmScheduleCovered($listed(), $every, array($every, array('dest' => 'backup/tank/data') + $copy))[0]['tank'], 'not one whose own copy is that copy'); +same(array('tank', 'tank/data', 'tank/docker', 'tank/san/kid', 'tank/vm'), zfsmScheduleCovered($listed(), $every, array($every, array('dest' => 'backup/data') + $copy))[0]['tank'], 'but one whose own copy is elsewhere'); +same(array('backup/old is in pool backup, where its copy would go', 'tank/data has a schedule of its own for that copy', ''), + array(zfsmScheduleCopiesNot($every, array(), 'backup/old'), zfsmScheduleCopiesNot($every, array(array('dest' => 'backup/tank/data') + $copy), 'tank/data'), zfsmScheduleCopiesNot($every, array(), 'tank/data')), 'why one is left out'); + +// Save: Copy Under must be a filesystem here; what is wrong is said, and nothing saved. +$wrong = array( + array('nowhere', 'hourly', '', '', 'Nothing was saved: nowhere does not exist on this server'), + array('tank/vm', 'hourly', '', '', 'Nothing was saved: tank/vm is a volume, which holds no datasets'), + array('bad name!', 'hourly', '', '', 'Nothing was saved: under Copy Under, name a dataset on this server, such as backup'), + array('', 'hourly', '', '', 'Nothing was saved: under Copy Under, name a dataset on this server, such as backup'), + array('backup', 'days', '', '9', 'Nothing was saved: set the time it starts at; tick at least one day'), +); +same(array_column($wrong, 4), array_map(function($try) use ($cfg) { return zfsmScheduleCopiesSave($try[0], $try[1], $try[2], $try[3], 0, $cfg)['why']; }, $wrong), 'what is wrong is said'); +same(null, $stored(), 'and nothing is saved'); +same(array('off', ''), array(zfsmScheduleShown('zfsmScheduleCopiesUnder')['every'], zfsmScheduleShown('zfsmScheduleCopiesUnder')['dest']), 'Settings shows Off'); + +// Every hour: at a minute Snapshot Every Dataset doesn't start at; the answer counts what it copies, and nothing is copied on saving. +zfsmScheduleSnapshotSave('', 'hourly', '', '', 0, $cfg); +$taken = array_values($stored())[0]['minute']; +$reset(); +$saved = zfsmScheduleCopiesSave('backup', 'hourly', '', '', 0, $cfg); +$job = array_values(array_filter($stored(), 'zfsmScheduleCopiesUnder'))[0]; +same(array('saved' => true, 'why' => '', 'said' => 'Saved: a copy of each of 6 dataset(s) under backup every hour at :'.sprintf('%02d', $job['minute']).'; it keeps all its snapshots'), $saved, 'saved, counted'); +same(array_replace($every, array('id' => $job['id'], 'minute' => $job['minute'])), $job, 'as the record'); +same(true, $job['minute'] !== $taken, 'at a minute of its own'); +same(array(), preg_grep('/^(snapshot|send|receive|create) /', $read('calls.log')), 'nothing copied on saving'); +same(array('hourly', 'backup/'), array(zfsmScheduleShown('zfsmScheduleCopiesUnder')['every'], zfsmScheduleShown('zfsmScheduleCopiesUnder')['dest']), 'Settings shows it'); +same(1, preg_match('/>Replicate Every Dataset under backup, every hour at :\d\d, keeps all its snapshots true, 'why' => '', 'said' => 'Replicate Every Dataset is off; the copies it made and their snapshots stay'), zfsmScheduleCopiesSave('', 'off', '', '', 0, $cfg), 'off'); +same(array(''), array_column($stored(), 'dest'), 'only the snapshot schedule stays'); + +// Runs: Replicate Every Dataset alone, keeping the newest 1. +$id = '0a1b2c3d'; +file_put_contents(zfsmScheduleFile(), json_encode(array(array('keep' => 1) + $every))); +$t = 1800000000; + +// The first, with tank's snapshot refused: the datasets below it wait, as their copies go inside its; usb is copied. One notification. +$reset(); +putenv('ZFSM_TEST_SNAP_FAIL=tank'); +same(1, zfsmScheduleRun($id, $cfg, $t), 'a run with a problem'); +putenv('ZFSM_TEST_SNAP_FAIL'); +same(array('backup/usb'), array_values(preg_grep('~^backup/(?!old$)~', $datasets())), 'only usb is copied'); +same(array('warning: ZFS Replication'), $read('notify.log'), 'one notification'); +same(1, preg_match('/^warning: Scheduled copies under backup: 1 of 6 dataset\(s\) copied\. Not copied, as the copy of a dataset above it is missing \(tank\/data, tank\/docker, tank\/san\/kid, and 1 more\)\. Replication of tank to backup\/tank not started: .*out of space\. Later problems/', $read('notify-detail.log')[0] ?? ''), 'naming what waits and why'); + +// The next: parents first, a holder in place of tank/san (left to Sanoid), each copy read-only and unmounted; tagged for the record. +$reset(); +same(0, zfsmScheduleRun($id, $cfg, $t + 3600), 'the next run works'); +$n = $name($t + 3600); +same(array('backup/usb', 'backup/tank', 'backup/tank/data', 'backup/tank/docker', 'backup/tank/san', 'backup/tank/san/kid', 'backup/tank/vm'), array_values(preg_grep('~^backup/(?!old$)~', $datasets())), 'in order, parents first'); +same(array('create -o canmount=off -o readonly=on -o zfs.master:holder=on backup/tank/san'), array_values(preg_grep('/^create /', $read('calls.log'))), 'one holder'); +same(array(true, false, false), array(zfsmReplicaHolder(array('server' => ''), 'backup/tank/san'), zfsmReplicaHolder(array('server' => ''), 'backup/tank/san/kid'), zfsmReplicaHolder(array('server' => ''), 'backup/tank')), 'a holder is one by its own property, not one inherited'); +same(array('normal: Scheduled copies under backup: 6 of 6 dataset(s) copied. It works again after 1 run(s) with a problem.'), $read('notify-detail.log'), 'and says so'); +same(array("tank/data@$n" => $tag), $snaps('tank/data'), 'its snapshot is tagged for the record'); +same(1, $logged('/ Schedule '.$id.': Scheduled copies under backup: 6 of 6 dataset\(s\) copied$/'), 'logged'); +same(array(), preg_grep('/receive .*-F/', $read('calls.log')), 'never with -F'); + +// The one after sends changes; Keep leaves the newest 1 of the record's snapshots on each dataset. +$reset(); +same(0, zfsmScheduleRun($id, $cfg, $t + 7200), 'a run of changes'); +$m = $name($t + 7200); +same(array("tank/data@$m" => $tag), $snaps('tank/data'), 'Keep leaves the newest'); +same(array("backup/tank/san/kid@$m"), array_keys($snaps('backup/tank/san/kid')), 'in the copy too'); +same(6, count(preg_grep('/^send -P -v -L -i /', $read('calls.log'))), 'each sends its changes'); +same(array(), $read('notify.log'), 'only logged'); + +// tank/san no longer Sanoid's: its copy's place holds a holder, which it can't replace; tank/san/kid goes on. +$pool('marks', array_values(array_filter($read('pool/marks'), function($line) { return strpos($line, 'tank/san@autosnap_') !== 0; }))); +$reset(); +same(1, zfsmScheduleRun($id, $cfg, $t + 10800), 'a run with the holder in the way'); +$o = $name($t + 10800); +same(1, preg_match('/^warning: Scheduled copies under backup: 6 of 7 dataset\(s\) copied\. Replication of tank\/san to backup\/tank\/san not started: backup\/tank\/san only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them/', $read('notify-detail.log')[0] ?? ''), 'says so'); +same(array(), $snaps('tank/san'), 'the snapshot it took for it goes'); +same(array("backup/tank/san/kid@$o"), array_keys($snaps('backup/tank/san/kid')), 'the one below is copied'); + +// Other replications of tank/data and tank/docker still running: the run's waits share its budget, so the second has none left. +$fresh(); +$reset(); +@unlink(zfsmScheduleStateFile($id)); +$locks = array_map(function($dataset) { return zfsmJobLock(zfsmJobLockFile($dataset, 'replicate')); }, array('tank/data', 'tank/docker')); +$started = time(); +same(1, zfsmScheduleRun($id, $cfg, $t + 14400, 2), 'a run that waits'); +array_map('zfsmJobUnlock', $locks); +same(true, time() - $started < 5, 'in all, about its budget'); +same(1, preg_match('/^warning: Scheduled copies under backup: 4 of 6 dataset\(s\) copied\. Replication of tank\/data to backup\/tank\/data not started: another replication of this dataset was still running after this run waited 1 minute\(s\) for it; Replication of tank\/docker to backup\/tank\/docker not started: another replication of this dataset was still running, and the run had no time left to wait for it/', $read('notify-detail.log')[0] ?? ''), 'each says so'); + +// A run while the one before still goes: none made, notified. +$reset(); +@unlink(zfsmScheduleStateFile($id)); +$held = zfsmJobLock(zfsmJobDir('schedule').'/'.$id.'.lock'); +same(1, zfsmScheduleRun($id, $cfg, $t + 18000), 'a run while the one before goes'); +zfsmJobUnlock($held); +same(array(array('warning: ZFS Replication'), 1), array($read('notify.log'), preg_match('/^warning: Scheduled copies not made: the run before is still going/', $read('notify-detail.log')[0] ?? '')), 'notified'); + +// Copy Under gone: none made; a later problem, so only logged. +$pool('datasets', array_values(array_filter($read('pool/datasets'), function($line) { return strpos($line, 'backup') !== 0; }))); +$reset(); +same(1, zfsmScheduleRun($id, $cfg, $t + 21600), 'a run with Copy Under gone'); +same(array(), $read('notify.log'), 'a later problem is only logged'); +same(1, $logged('/ Schedule '.$id.': Scheduled copies not made: backup does not exist on this server$/'), 'saying why'); + +echo "schedule copies tests passed\n"; diff --git a/tests/schedule_snapshot_test.php b/tests/schedule_snapshot_test.php index 612d71f5..a3d07ff2 100644 --- a/tests/schedule_snapshot_test.php +++ b/tests/schedule_snapshot_test.php @@ -62,7 +62,7 @@ function same($expected, $actual, $message) { // Off with none saves none; Settings shows Off. same(array('saved' => true, 'why' => '', 'said' => 'Snapshot Every Dataset is off; the snapshots it took stay'), zfsmScheduleSnapshotSave('', 'off', '', '', 0, $cfg), 'off with none'); -same(array(array(), 'off'), array($stored(), zfsmScheduleSnapshotShown()['every']), 'saves no record'); +same(array(array(), 'off'), array($stored(), zfsmScheduleShown()['every']), 'saves no record'); // Saved again every hour: its id and minute stay, though another minute is free. zfsmScheduleSnapshotSave('', 'hourly', '', '', 0, $cfg); $first = array_values($stored())[0]; @@ -90,8 +90,8 @@ function same($expected, $actual, $message) { // Saved again, in place: its id and minute stay, and Keep changes. same('Saved: a snapshot of each of 5 dataset(s) every hour at :30; it keeps all it takes', zfsmScheduleSnapshotSave('', 'hourly', '', '', 0, $cfg)['said'], 'Keep changes in place'); same(array_replace($every, array('id' => $id, 'keep' => 0)), zfsmScheduleRead()['valid'][$id], 'the same record'); -same(array(60, 'hourly'), array(count(zfsmScheduleRead()['valid']), zfsmScheduleSnapshotShown()['every']), 'no second one, and Settings shows it'); -same(array(3, 0), array(zfsmScheduleSnapshotShown()['hour'], zfsmScheduleSnapshotShown()['minute']), 'with Starts At at Off\'s 03:00, not the :30 it runs at'); +same(array(60, 'hourly'), array(count(zfsmScheduleRead()['valid']), zfsmScheduleShown()['every']), 'no second one, and Settings shows it'); +same(array(3, 0), array(zfsmScheduleShown()['hour'], zfsmScheduleShown()['minute']), 'with Starts At at Off\'s 03:00, not the :30 it runs at'); // At a time a copy starts: refused, the copy named; the record stays as it was. file_put_contents(zfsmScheduleFile(), json_encode(array(array_replace($copy, array('id' => '0b0b0b0b', 'every' => 'days', 'minute' => 7, 'hour' => 3)), array_replace($every, array('id' => $id))))); $wrong = array( @@ -106,7 +106,7 @@ function same($expected, $actual, $message) { same('Saved: a snapshot of each of 5 dataset(s) every Monday and Saturday at 04:00; it keeps the newest 3 scheduled snapshots of each dataset', zfsmScheduleSnapshotSave('', 'days', '04:00', '6,1', 3, $cfg)['said'], 'on days'); same(array_replace($every, array('id' => $id, 'every' => 'days', 'minute' => 0, 'hour' => 4, 'days' => array(1, 6), 'keep' => 3)), zfsmScheduleRead()['valid'][$id], 'saved so'); same(1, substr_count((string)file_get_contents(zfsmScheduleCronFile()), "0 4 * * 1,6 $script run $id &> /dev/null\n"), 'its cron line'); -$shown = zfsmScheduleSnapshotShown(); +$shown = zfsmScheduleShown(); same(array('days', 4, 0, array(1, 6)), array($shown['every'], $shown['hour'], $shown['minute'], $shown['days']), 'and Settings shows its time and days'); file_put_contents(zfsmScheduleFile(), json_encode(array(array_replace($every, array('id' => $id))))); @@ -228,7 +228,7 @@ function same($expected, $actual, $message) { zfsmScheduleSnapshotSave('tank/data', 'days', '05:00', '1', 1, $cfg), 'a dataset\'s own, counted'); $ids = array_column($stored(), 'id', 'dataset'); same(array_replace($every, array('id' => $ids['tank/data'], 'dataset' => 'tank/data', 'every' => 'days', 'minute' => 0, 'hour' => 5, 'days' => array(1), 'keep' => 1)), zfsmScheduleRead()['valid'][$ids['tank/data']], 'as its record'); -same(array(1, 'off'), array(substr_count($cron(), "0 5 * * 1 $script run {$ids['tank/data']} &> /dev/null\n"), zfsmScheduleSnapshotShown()['every']), 'its cron line; Snapshot Every Dataset stays Off'); +same(array(1, 'off'), array(substr_count($cron(), "0 5 * * 1 $script run {$ids['tank/data']} &> /dev/null\n"), zfsmScheduleShown()['every']), 'its cron line; Snapshot Every Dataset stays Off'); // The default at the same time: two snapshot schedules never take one dataset's, so may share a minute. same('Saved: a snapshot of each of 4 dataset(s) every Monday at 05:00; it keeps all it takes', zfsmScheduleSnapshotSave('', 'days', '05:00', '1', 0, $cfg)['said'], 'the default counts all but tank/data'); $ids = array_column($stored(), 'id', 'dataset'); @@ -340,7 +340,7 @@ function same($expected, $actual, $message) { $own = array_column($stored(), 'dataset'); sort($own); same(array(array('tank', 'tank/data', 'tank/docker'), 'off', 'Snapshot Every Dataset, which is off: no scheduled snapshots'), - array($own, zfsmScheduleSnapshotShown()['every'], $preview('backup')['inherited']), 'leaves the datasets\' own'); + array($own, zfsmScheduleShown()['every'], $preview('backup')['inherited']), 'leaves the datasets\' own'); same('Saved: tank follows Snapshot Every Dataset, which is off: no scheduled snapshots', zfsmScheduleSnapshotSave('tank', 'default', '', '', 0, $cfg)['said'], 'Default with nothing above'); // Rename: the own schedules of it and below follow it, ids kept, and one left at the new name gives way, its count too. diff --git a/tests/settings_send_test.cjs b/tests/settings_send_test.cjs index b02855fa..a4caeb31 100644 --- a/tests/settings_send_test.cjs +++ b/tests/settings_send_test.cjs @@ -24,7 +24,12 @@ const {window} = new JSDOM('Days:
' + '
Starts At:
' + '
Snapshots To Keep:
' - + '
 
' + + '
 
' + + '
Replicate Every Dataset:
' + + '
Copy Under:
' + + '
Days:
' + + '
Starts At:
Snapshots To Keep:
' + + '
 
' + 'tank/data to backup/data', {url: 'http://localhost/', runScripts: 'outside-only'}); window.eval(fs.readFileSync(jquery, 'utf8')); @@ -201,23 +206,37 @@ assert.deepEqual(shown(), ['none/none', 'none/none', '/'], 'Every hour shows Kee const color = () => window.document.getElementById('zfsm-snapsched-status').style.color; // Save sends what the rows say, the ticked days in their order; it is off while it saves, and the answer's list replaces Settings'. $('#zfsm-snapsched-every').val('days').trigger('change'); -window.zfsmSnapshotsSave(); +window.zfsmEverySave('zfsm-snapsched', 'schedulesnapshots'); const snapshots = posts.pop(); assert.deepEqual({...snapshots.data}, {cmd: 'schedulesnapshots', every: 'days', time: '04:00', days: '1,0', keep: '3'}); assert.deepEqual([$('#zfsm-snapsched-save').prop('disabled'), text('zfsm-snapsched-status')], [true, 'Saving…'], 'Save is off while it saves'); snapshots.done({said: 'Saved: a snapshot of each of 5 dataset(s)', schedules_html: 'Snapshot Every Dataset'}); assert.deepEqual([$('#zfsm-snapsched-save').prop('disabled'), text('zfsm-snapsched-status'), color(), text('zfsm-schedules')], [false, 'Saved: a snapshot of each of 5 dataset(s)', '', 'Snapshot Every Dataset'], 'saved: says what, and the list shows it'); -window.zfsmSnapshotsSave(); +window.zfsmEverySave('zfsm-snapsched', 'schedulesnapshots'); posts.pop().done({error: 'Nothing was saved: tick at least one day'}); assert.deepEqual([$('#zfsm-snapsched-save').prop('disabled'), text('zfsm-snapsched-status'), color(), text('zfsm-schedules')], [false, 'Nothing was saved: tick at least one day', 'rgb(226, 40, 40)', 'Snapshot Every Dataset'], 'a refusal says why in red and keeps the list'); -window.zfsmSnapshotsSave(); +window.zfsmEverySave('zfsm-snapsched', 'schedulesnapshots'); posts.pop().done({said: 'Saved: x', error: 'The schedules were saved, but Unraid\'s crontab could not be rebuilt (y)', schedules_html: 'z'}); assert.deepEqual([text('zfsm-snapsched-status'), color(), text('zfsm-schedules')], ['Saved: x. The schedules were saved, but Unraid\'s crontab could not be rebuilt (y)', 'rgb(226, 40, 40)', 'z'], 'saved with a crontab that was not rebuilt: both, in red'); -window.zfsmSnapshotsSave(); +window.zfsmEverySave('zfsm-snapsched', 'schedulesnapshots'); posts.pop().done({succeeded: {}, failed: {request: 'Unknown or missing command'}}); assert.deepEqual([$('#zfsm-snapsched-save').prop('disabled'), text('zfsm-snapsched-status'), text('zfsm-schedules')], [false, 'Unknown or missing command', 'z'], 'as does a backend failure'); +// Replicate Every Dataset: its own rows by its prefix, Copy Under with Keep; on load as saved (Every hour), and Save sends Copy Under too. +const copyShown = () => ['zfsm-copysched-under', 'zfsm-copysched-days', 'zfsm-copysched-time', 'zfsm-copysched-keep'].map(id => window.document.getElementById(id).closest('dd').style.display); +assert.deepEqual(copyShown(), ['', 'none', 'none', ''], 'Every hour on load shows Copy Under and Keep only'); +$('#zfsm-copysched-every').val('off').trigger('change'); +assert.deepEqual(copyShown(), ['none', 'none', 'none', 'none'], 'Off shows none of its other rows'); +assert.deepEqual(shown(), ['/', '/', '/'], 'and leaves Snapshot Every Dataset\'s rows as they were'); +$('#zfsm-copysched-every').val('days').trigger('change'); +window.zfsmEverySave('zfsm-copysched', 'schedulecopies'); +const copies = posts.pop(); +assert.deepEqual({...copies.data}, {cmd: 'schedulecopies', every: 'days', time: '02:30', days: '3', keep: '7d', under: 'backup'}); +assert.deepEqual([$('#zfsm-copysched-save').prop('disabled'), text('zfsm-copysched-status'), text('zfsm-snapsched-status')], [true, 'Saving…', 'Unknown or missing command'], 'its Save and answer are its own'); +copies.done({said: 'Saved: a copy of each of 4 dataset(s) under backup', schedules_html: 'w'}); +assert.deepEqual([$('#zfsm-copysched-save').prop('disabled'), text('zfsm-copysched-status'), text('zfsm-schedules')], [false, 'Saved: a copy of each of 4 dataset(s) under backup', 'w']); + console.log('settings send tests passed on jQuery ' + window.$.fn.jquery); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index a47c9efe..4d089b39 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -15,7 +15,8 @@ $zfsm_saved = parse_plugin_cfg($plugin, true); $zfsm_cfg = loadConfig($zfsm_saved); // Keep Replicas' field shows what applies (empty: all), so a saved value it can't read never holds up Apply; the warning names it. $zfsm_keep_warning = zfsmKeepReplicaWarning((string)($zfsm_saved['general']['replica_keep'] ?? '')); -$zfsm_snapsched = zfsmScheduleSnapshotShown(); +$zfsm_snapsched = zfsmScheduleShown(); +$zfsm_copysched = zfsmScheduleShown('zfsmScheduleCopiesUnder'); $zfsm_version = zfsmPluginVersion("/var/log/plugins/$plugin.plg"); ?> @@ -332,13 +333,47 @@ _(Snapshots To Keep)_: :   -: +: + +_(Replicate Every Dataset)_: +: + +:zc_settings_copysched_plug: +> Copies every dataset in every pool into the dataset you name under **Copy Under**, on this server, each copy named for its dataset: with `backup`, `cache/appdata` is copied to `backup/cache/appdata`. It runs every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. Save changes it, and the answer says how many datasets it copies. Off stops it; the copies and the snapshots it made stay. +> +> - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full. The copies are read-only and not mounted. +> - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy. +> - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. When the copy of the dataset above one is missing because it failed, that one waits for the next run. +> - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. +> - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. +> - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. +> - Each run sends one notification at most: what it copied and its first problems. The first run with a problem sends it; later ones are logged in `/var/log/zfs.master.log` until a run works again, and that run's notification says so. The log has each dataset's replication. +:end + +_(Copy Under)_: +: + +_(Days)_: +: + +_(Starts At)_: +: + +_(Snapshots To Keep)_: +: + +  +: _(Schedules)_: : :zc_settings_schedules_plug: -> Snapshot Every Dataset, set above, each dataset's own snapshot schedule (set under **Actions**, **Schedules** in the Main tab; Remove here sets it back to Default), and the replications that run on their own. To add one of those, choose **Actions**, **Replicate** on a dataset in the Main tab and pick how often under Repeat: it runs then, and again every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. This list shows when. +> Snapshot Every Dataset and Replicate Every Dataset, set above, each dataset's own snapshot schedule (set under **Actions**, **Schedules** in the Main tab; Remove here sets it back to Default), and the replications that run on their own. To add one of those, choose **Actions**, **Replicate** on a dataset in the Main tab and pick how often under Repeat: it runs then, and again every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. This list shows when. > > - Each run takes a new snapshot, named by **Snapshots Pattern**, and sends the changes since the last one. If the run before it stopped part-way, it carries that on first. > - **Snapshots to keep** (set in Replicate; Remove the schedule and add it again to change it): from its second run on, once a run lands, only those it keeps of the snapshots this copy's schedules took stay, as **Snapshots To Keep** above reads it (such as `24 7d 4w 12m`), up to 50 removed a run. In a copy on this server the same snapshots go first; one held or cloned, there or here, stays on both. Snapshots made by hand, by Snapshot Every Dataset or by a schedule of another copy are never counted or removed. The one just sent counts but always stays: the next run sends the changes since it. A copy on another server keeps all of them, unless that server's **Keep Replicas** keeps fewer. Empty keeps all. @@ -482,28 +517,34 @@ function zfsmScheduleRemove(id) { if (answer.schedules_html !== undefined) $('#zfsm-schedules').html(answer.schedules_html); }); } -// Snapshot Every Dataset: Days and Starts At for days of the week, Snapshots To Keep for any repeat. -function zfsmSnapshotsSync() { - var every = $('#zfsm-snapsched-every').val(); +// Snapshot and Replicate Every Dataset, by their rows' id prefix: Days and Starts At for days of the week, the rest for any repeat. +function zfsmEverySync(prefix) { + var every = $('#' + prefix + '-every').val(); // Not .toggle(): jQuery 2 (Unraid 6.9) writes display:block over the theme's own display. var show = function(id, shown) { var dd = $(id).closest('dd'); dd.add(dd.prev('dt')).css('display', shown ? '' : 'none'); }; - show('#zfsm-snapsched-days', every === 'days'); - show('#zfsm-snapsched-time', every === 'days'); - show('#zfsm-snapsched-keep', every !== 'off'); + show('#' + prefix + '-days', every === 'days'); + show('#' + prefix + '-time', every === 'days'); + show('#' + prefix + '-keep', every !== 'off'); + show('#' + prefix + '-under', every !== 'off'); } -function zfsmSnapshotsSave() { +function zfsmEverySave(prefix, cmd) { // Off while it runs: a second click would save the same again. - var button = $('#zfsm-snapsched-save').prop('disabled', true); - var status = $('#zfsm-snapsched-status').css('color', '').text('Saving…'); - var days = $('.zfsm-snapsched-day:checked').map(function() { return this.value; }).get().join(','); - zfsmSendPost({cmd: 'schedulesnapshots', every: $('#zfsm-snapsched-every').val(), time: $('#zfsm-snapsched-time').val(), days: days, keep: $('#zfsm-snapsched-keep').val()}, function(answer) { + var button = $('#' + prefix + '-save').prop('disabled', true); + var status = $('#' + prefix + '-status').css('color', '').text('Saving…'); + var days = $('.' + prefix + '-day:checked').map(function() { return this.value; }).get().join(','); + var post = {cmd: cmd, every: $('#' + prefix + '-every').val(), time: $('#' + prefix + '-time').val(), days: days, keep: $('#' + prefix + '-keep').val()}; + var under = $('#' + prefix + '-under'); + if (under.length) post.under = under.val(); + zfsmSendPost(post, function(answer) { button.prop('disabled', false); status.css('color', answer.error ? '#e22828' : '').text([answer.said, answer.error].filter(Boolean).join('. ')); if (answer.schedules_html !== undefined) $('#zfsm-schedules').html(answer.schedules_html); }); } -$('#zfsm-snapsched-every').on('change', zfsmSnapshotsSync); -zfsmSnapshotsSync(); +['zfsm-snapsched', 'zfsm-copysched'].forEach(function(prefix) { + $('#' + prefix + '-every').on('change', function() { zfsmEverySync(prefix); }); + zfsmEverySync(prefix); +}); function zfsmSendTest(name, id) { var status = $(document.getElementById(id)).css('color', '').text('Asking…'); zfsmSendPost({cmd: 'replicatest', name: name}, function(answer) { diff --git a/zfs.master/backend/ZFSMAdmin.php b/zfs.master/backend/ZFSMAdmin.php index 67beb8e7..236d899e 100644 --- a/zfs.master/backend/ZFSMAdmin.php +++ b/zfs.master/backend/ZFSMAdmin.php @@ -234,6 +234,12 @@ function zfsmRequireDestructiveMode($config, $subject) { echo zfsmJson(array_filter(array('said' => $saved['said'], 'error' => $saved['why'])) + ($saved['saved'] ? array('schedules_html' => zfsmSchedulesHtml($zfsm_cfg)) : array())); break; + case 'schedulecopies': + // Settings' Replicate Every Dataset, answered as schedulesnapshots is. + $saved = zfsmScheduleCopiesSave(zfsmPostString('under'), zfsmPostString('every'), zfsmPostString('time'), zfsmPostString('days'), zfsmPostKeep(), $zfsm_cfg); + echo zfsmJson(array_filter(array('said' => $saved['said'], 'error' => $saved['why'])) + ($saved['saved'] ? array('schedules_html' => zfsmSchedulesHtml($zfsm_cfg)) : array())); + break; + case 'scheduleremove': // Saved with only its cron step failed, the list changed all the same: the answer has both. $removed = zfsmScheduleRemove(zfsmPostString('id')); diff --git a/zfs.master/include/ZFSMDiagNames.php b/zfs.master/include/ZFSMDiagNames.php index 88bd256c..14d0657e 100644 --- a/zfs.master/include/ZFSMDiagNames.php +++ b/zfs.master/include/ZFSMDiagNames.php @@ -14,8 +14,9 @@ define('ZFSM_DIAG_FAILED', 'not started: |failed(?:: | after | with exit code |$)'); // Keep's words after a copy's name (ZFSMKeep.php); in Keep's lines a snapshot's name ends at ( or ;, which no name holds. define('ZFSM_DIAG_KEEP', '(?<=: )Keep could not '); -// A dataset's own snapshot schedule: why a run takes none of it (zfsmScheduleSnapshotNone), and whose datasets it took. -define('ZFSM_DIAG_SCHEDULE', 'not taken: |is (?:gone|read-only|mounted legacy)$|has Sanoid or Syncoid snapshots *$|and below$'); +// A dataset's own snapshot schedule: why a run takes none of it (zfsmScheduleSnapshotNone), and whose datasets it took; copies +// under a dataset (ZFSMScheduleCopies.php): how many a run copied, a holder it could not make, one in a copy's place. +define('ZFSM_DIAG_SCHEDULE', 'not taken: |is (?:gone|read-only|mounted legacy)$|has Sanoid or Syncoid snapshots *$|and below$|\\d+ of \\d+ dataset$|could not be made: |only holds the copies '); define('ZFSM_DIAG_AFTER', ZFSM_DIAG_DONE.'|'.ZFSM_DIAG_FAILED.'|'.ZFSM_DIAG_KEEP.'|'.ZFSM_DIAG_SCHEDULE.'|is not inside |(?:to [^/\s]+(?: [^/\s]+)*? )?on server |\((?:in full|changes since|carried on)|\| '); // And after a folder's path: an action's result (zfsmActionLogLines), Convert's fallback to rsync and a nested mount's refusal (backend/ZFSMOperations.php). define('ZFSM_DIAG_FOLDER_AFTER', ' succeeded$| failed(?:: |, so rsync copies it: )|; unmount or move it first$'); @@ -248,7 +249,7 @@ function zfsmDiagAnonPath($text, $at, $ends = "\t\n") { // Whether $path is one the plugin makes in its own places, whole: a run's work folder and its files, a job's folder and its locks, // the plugin's lock and alert state, a package record, its files on the flash drive. Nothing else in or beside them is its own. function zfsmDiagOwnName($path) { - $jobs = '(?:rewrite|replicate|replica-serve|replica-send|schedule|diagnostics)(?:/(?:(?:\d+(?:\.\d+)?|pack|run|start|settings)\.lock|state\.json(?:\.tmp)?|[0-9a-f]{8}\.json|browser\.json|ssh-[0-9a-f]{12}(?:/(?:id_ed25519|known_hosts|cm))?))?'; + $jobs = '(?:rewrite|replicate|replica-serve|replica-send|schedule|diagnostics)(?:/(?:(?:\d+(?:\.\d+)?|pack|run|start|settings|[0-9a-f]{8})\.lock|state\.json(?:\.tmp)?|[0-9a-f]{8}\.json|browser\.json|ssh-[0-9a-f]{12}(?:/(?:id_ed25519|known_hosts|cm))?))?'; // zfsmJobDir('') is /var/run/zfs.master. but in a test: listed after it, the fuller entry wins where they are one. $names = array( zfsmStatePath('/tmp/zfs.master-diag.') => '[0-9a-f]{16}(?:/(?:(?:plugin|zfs)/[a-z0-9._-]+|unraid/(?:context|diagnostics)\.txt|(?:names|README|collectors|summary)\.txt))?', diff --git a/zfs.master/include/ZFSMDiagPlugin.php b/zfs.master/include/ZFSMDiagPlugin.php index c6c254c0..76d1d4f5 100644 --- a/zfs.master/include/ZFSMDiagPlugin.php +++ b/zfs.master/include/ZFSMDiagPlugin.php @@ -244,7 +244,9 @@ function zfsmDiagReplicaSchedules($ctx) { foreach ($read['valid'] ?? array() as $id => $job) { zfsmDiagKnowWord($job['server']); $dest = $job['server'] === '' ? zfsmDiagNamed($job['dest']) : zfsmDiagToken($job['dest']).' on server '.$job['server']; - $what = !zfsmScheduleSnapshotsOnly($job) ? array(zfsmDiagNamed($job['dataset']), $dest) : array($job['dataset'] === '' ? '(every dataset)' : zfsmDiagNamed($job['dataset']), '(snapshots only)'); + $whose = $job['dataset'] === '' ? '(every dataset)' : zfsmDiagNamed($job['dataset']); + $under = $job['every'] === 'none' ? '(no copies)' : zfsmDiagNamed(rtrim($job['dest'], '/')).'/ (each dataset under it)'; + $what = zfsmScheduleCopiesUnder($job) ? array($whose, $under) : (zfsmScheduleSnapshotsOnly($job) ? array($whose, '(snapshots only)') : array(zfsmDiagNamed($job['dataset']), $dest)); $runs = zfsmScheduleRepeats($job['every']); $failures = zfsmScheduleProblems($job); $rows .= implode("\t", array_merge(array($id), $what, array(zfsmScheduleWhen($job), $job['keep'], $failures)))."\n"; @@ -406,10 +408,10 @@ function zfsmDiagPublicHalf($dir, $private, $public) { return zfsmDiagWhole('file_get_contents', $dir.'/id_ed25519.pub') === false ? zfsmDiagUnread('unreadable') : 'there, but not a usable key'; } -// How many datasets each snapshot schedule that runs takes now, by id, from reads under a limit of their own so a hung pool costs -// only this line; '' with none. +// How many datasets each snapshot schedule and copies under a dataset that runs covers now, by id, from reads under a limit of their +// own so a hung pool costs only this line; '' with none. function zfsmDiagScheduleCovers($jobs, $cfg) { - $runs = array_filter($jobs, function($job) { return zfsmScheduleSnapshotsOnly($job) && zfsmScheduleRepeats($job['every']); }); + $runs = array_filter($jobs, function($job) { return (zfsmScheduleSnapshotsOnly($job) || zfsmScheduleCopiesUnder($job)) && zfsmScheduleRepeats($job['every']); }); if (!$runs) return ''; $listed = zfsmScheduleCoverRead($cfg, 5, array(), function($wait, $command, $args) { return zfsmDiagRead($command, $args, $wait); }); $counts = array(); @@ -420,8 +422,8 @@ function zfsmDiagScheduleCovers($jobs, $cfg) { // Each minds only its own errors (zfsmScheduleCovered): a later one's '' must not hide an earlier one's. if ($unread === '') $unread = $one; } - if ($unread !== '' && !zfsmScheduleListed($listed)[0]) return 'datasets each snapshot schedule takes now: (could not be read: '.$unread.')'."\n"; - return 'datasets each snapshot schedule takes now: '.implode(', ', $counts).($unread !== '' ? ' (not every dataset could be read: '.$unread.')' : '')."\n"; + if ($unread !== '' && !zfsmScheduleListed($listed)[0]) return 'datasets each schedule of snapshots or of copies under a dataset covers now: (could not be read: '.$unread.')'."\n"; + return 'datasets each schedule of snapshots or of copies under a dataset covers now: '.implode(', ', $counts).($unread !== '' ? ' (not every dataset could be read: '.$unread.')' : '')."\n"; } // What the Main tab's first read returns, logging a failure as that read does. It asks about every pool, so it is skipped once diff --git a/zfs.master/include/ZFSMKeep.php b/zfs.master/include/ZFSMKeep.php index f36ae054..6b9c65e3 100644 --- a/zfs.master/include/ZFSMKeep.php +++ b/zfs.master/include/ZFSMKeep.php @@ -144,8 +144,8 @@ function zfsmKeepRemove($removable, $copies, $why) { } // The log and notifications keep 500 characters of a line: the first few of $items, and how many more. -function zfsmKeepFirst($items) { - return implode('; ', array_slice($items, 0, 3)).(count($items) > 3 ? '; and '.(count($items) - 3).' more' : ''); +function zfsmKeepFirst($items, $join = '; ') { + return implode($join, array_slice($items, 0, 3)).(count($items) > 3 ? $join.'and '.(count($items) - 3).' more' : ''); } // What $stayed (each a snapshot and why) says in a log line or notification, '' when none; the next $then tries again. diff --git a/zfs.master/include/ZFSMReplicaPlan.php b/zfs.master/include/ZFSMReplicaPlan.php index 370cf575..64bfa447 100644 --- a/zfs.master/include/ZFSMReplicaPlan.php +++ b/zfs.master/include/ZFSMReplicaPlan.php @@ -128,6 +128,7 @@ function zfsmReplicatePlan($zdataset, $snapshot, $zdest, $with = array()) { if (!$missing) { list($held, $why) = zfsmReplicaTargetSnapshots($target, $zdest); if ($held === null) return $refuse('The snapshots of '.$where.' could not be read'.$because($why)); + if (!$held && zfsmReplicaHolder($target, $zdest)) return $refuse($where.' only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them'); if (!$held) return $refuse($where.' already exists and holds no snapshot, so it can\'t be updated without destroying it'); $newest = end($held); $base = null; diff --git a/zfs.master/include/ZFSMReplicaTarget.php b/zfs.master/include/ZFSMReplicaTarget.php index bba2c2f9..de111fa3 100644 --- a/zfs.master/include/ZFSMReplicaTarget.php +++ b/zfs.master/include/ZFSMReplicaTarget.php @@ -26,6 +26,23 @@ function zfsmReplicaTargetDest($target, $zdataset, $name) { return zfsmDatasetExists($parent, 'filesystem') ? array($name, '') : array(null, 'The copy goes inside '.$parent.', which doesn\'t exist'); } +// Marks a holder (zfsmReplicaHolderMake). The copies below one inherit it: only its own value (source local) makes one. +define('ZFSM_REPLICA_HOLDER', 'zfs.master:holder'); + +// Makes $zdest here an empty dataset, unmounted and read-only, that holds copies of datasets below the one whose copy it stands +// for; '' or why not. +function zfsmReplicaHolderMake($zdest) { + $made = runProcess('zfs', array('create', '-o', 'canmount=off', '-o', 'readonly=on', '-o', ZFSM_REPLICA_HOLDER.'=on', $zdest)); + return ($made['code'] ?? 1) === 0 ? '' : 'The dataset to hold the copies below '.$zdest.' could not be made: '.zfsmCommandError($made, 'zfs create failed'); +} + +// Whether $zdest on $target is a holder; false on a trusted server, which makes none, and when it can't be read. +function zfsmReplicaHolder($target, $zdest) { + if ($target['server'] !== '') return false; + $read = runProcess('zfs', array('get', '-H', '-p', '-o', 'source', ZFSM_REPLICA_HOLDER, $zdest)); + return ($read['code'] ?? 1) === 0 && trim((string)$read['output']) === 'local'; +} + // The copy as messages name it. "on server " is how diagnostics learn a server's name (zfsmDiagLearnPools). function zfsmReplicaTargetWhere($target, $zdest) { return $target['server'] === '' ? $zdest : $zdest.' on server '.$target['server']; diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index 32279474..672f017f 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -137,12 +137,13 @@ function zfsmReplicaMark($zdataset, $plan) { // The background run, here or to $server (opened under the lock): one per dataset at a time, its progress on the row, a notification at the end, or why it didn't start. // $take: the run takes $snapshot first, under its lock, and removes it again unless a copy holds it or a stopped run may need it. // $started runs once the plan holds, before zfs send; until then the reason a run doesn't start also goes to the starting page. -// $schedule, for a scheduled run (zfsmScheduleRun): its 'notify'(title, text, importance), the seconds to 'wait' out another run, and no full send to a copy that is gone. +// $schedule, for a scheduled run (zfsmScheduleRun): its 'notify'(title, text, importance), the seconds to 'wait' out another run, and no full send to a copy that is gone; +// 'tag' for a schedule of many datasets (zfsmScheduleCopiesRun), which names its holds and Keep instead of the copy's. // $keep, for any run of a schedule: the snapshot it takes is tagged for the copy (zfsmKeepTag), and once it lands only those of // them $keep keeps stay (0: all, zfsmKeepPrune). function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = false, $server = '', $schedule = null, $keep = null) { $title = 'ZFS Replication'; - $tag = zfsmKeepTag($server, $zdest); + $tag = $schedule['tag'] ?? zfsmKeepTag($server, $zdest); $notify = $schedule['notify'] ?? function($title, $text, $type) { zfsnotify($title, $text, '', $type); }; $to = zfsmReplicaTargetWhere(array('server' => $server), $zdest); $not_started = function($why) use ($zdataset, $to, $title, $started, $notify) { @@ -157,12 +158,13 @@ function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = return $not_started($why); }; $lock_file = zfsmJobLockFile($zdataset, 'replicate'); - if ($lock_file === null) return $not_started(zfsmDatasetMissing($zdataset) === true ? 'it no longer exists'.($schedule !== null ? '; remove its schedule in Settings' : '') : 'its guid could not be read'); + if ($lock_file === null) return $not_started(zfsmDatasetMissing($zdataset) === true ? 'it no longer exists'.($schedule !== null && !isset($schedule['tag']) ? '; remove its schedule in Settings' : '') : 'its guid could not be read'); $lock = zfsmJobLock($lock_file); for ($until = time() + ($schedule['wait'] ?? 0); $lock === false && time() < $until; $lock = zfsmJobLock($lock_file)) sleep(max(1, min(10, $until - time()))); if ($lock === null) return $not_started('its lock file could not be opened'); // No page starts a scheduled run again: one that waited in vain is a problem. - if ($lock === false && $schedule !== null) return $not_started('another replication of this dataset was still running after this run waited '.(int)ceil(($schedule['wait'] ?? 0) / 60).' minute(s) for it'); + $waited = (int)ceil(($schedule['wait'] ?? 0) / 60); + if ($lock === false && $schedule !== null) return $not_started('another replication of this dataset was still running'.($waited > 0 ? ' after this run waited '.$waited.' minute(s) for it' : ', and the run had no time left to wait for it')); // Another run holds it: two starts at once leave one replication, and that one sends the notification. if ($lock === false) { zfsmLog('Replication of '.$zdataset.' to '.$to.' not started: another replication of this dataset is running', LOG_NOTICE); diff --git a/zfs.master/include/ZFSMSchedule.php b/zfs.master/include/ZFSMSchedule.php index 7a6932bc..343a5fff 100644 --- a/zfs.master/include/ZFSMSchedule.php +++ b/zfs.master/include/ZFSMSchedule.php @@ -1,10 +1,11 @@ = 0 && $value <= $top; }; return $what && $number($job['minute'] ?? null, 59) && $number($job['hour'] ?? null, 23) && zfsmScheduleDaysValid($job['days'] ?? null) && zfsmKeepValid($job['keep'] ?? 0); } @@ -134,17 +135,20 @@ function zfsmScheduleFreeMinute($jobs, $job, $prefer = null) { return $free ? $free[array_rand($free)] : random_int(0, 59); } -// Whether $a and $b can snapshot one dataset (so not in one minute or name): two copies of it, or a copy and the snapshot schedule -// of it, of one above it, or the default. Each dataset follows one snapshot schedule (zfsmScheduleFollowed); None takes none. +// Whether $a and $b can snapshot one dataset (so not in one minute or name): both of it, or one that covers datasets below its own +// (snapshots, copies under a dataset) of one above it or the default. Each dataset follows one of each kind (zfsmScheduleFollowed); +// None takes none. function zfsmScheduleShare($a, $b) { if (!zfsmScheduleRepeats($a['every']) || !zfsmScheduleRepeats($b['every'])) return false; - if (zfsmScheduleSnapshotsOnly($a) === zfsmScheduleSnapshotsOnly($b)) return !zfsmScheduleSnapshotsOnly($a) && $a['dataset'] === $b['dataset']; - list($snapshots, $copy) = zfsmScheduleSnapshotsOnly($a) ? array($a, $b) : array($b, $a); - return $snapshots['dataset'] === '' || zfsmUnderAny($copy['dataset'], array($snapshots['dataset'])); + $covers = function($job) { return zfsmScheduleSnapshotsOnly($job) || zfsmScheduleCopiesUnder($job); }; + if ($covers($a) && $covers($b) && zfsmScheduleKindOf($a) === zfsmScheduleKindOf($b)) return false; + $in = function($job, $other) use ($covers) { return $covers($job) && ($job['dataset'] === '' || zfsmUnderAny($other['dataset'], array($job['dataset']))); }; + return $a['dataset'] === $b['dataset'] || $in($a, $b) || $in($b, $a); } // What a schedule does, inside a sentence. function zfsmScheduleWhat($job) { + if (zfsmScheduleCopiesUnder($job)) return $job['dataset'] === '' ? 'Replicate Every Dataset' : 'the copies of '.$job['dataset'].' and below'; if (!zfsmScheduleSnapshotsOnly($job)) return 'the copy to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']); return $job['dataset'] === '' ? 'Snapshot Every Dataset' : 'the snapshot schedule of '.$job['dataset']; } @@ -230,6 +234,7 @@ function zfsmScheduleWhen($job) { function zfsmScheduleKeepText($job) { $kept = zfsmKeepWords($job['keep']); if (zfsmScheduleSnapshotsOnly($job)) return $kept !== '' ? 'keeps '.$kept.' scheduled snapshots of each dataset' : 'keeps all it takes'; + if (zfsmScheduleCopiesUnder($job)) return $kept !== '' ? 'keeps '.$kept.' of the snapshots of each dataset it copies' : 'keeps all its snapshots'; return $kept !== '' ? 'keeps '.$kept.' of its snapshots' : 'keeps all its snapshots'; } @@ -359,15 +364,16 @@ function zfsmScheduleRun($id, $cfg, $now, $wait = ZFSM_SCHEDULE_WAIT) { zfsmLog('Schedule '.$id.' not started: '.($job === null ? 'Settings has no such schedule' : 'it is None now')); return 0; } - $copy = !zfsmScheduleSnapshotsOnly($job); + $copy = !zfsmScheduleSnapshotsOnly($job) && !zfsmScheduleCopiesUnder($job); try { + if (zfsmScheduleCopiesUnder($job)) return zfsmScheduleCopiesRun($job, $read['valid'], $cfg, $now, $notify, $wait); if (!$copy) return zfsmScheduleSnapshotRun($job, $read['valid'], $cfg, $now, $notify); return zfsmRunReplicate($job['dataset'], zfsmNewSnapshotName($cfg, $now), $job['dest'], null, true, $job['server'], array('notify' => $notify, 'wait' => $wait), $job['keep']); } catch (Throwable $error) { // Cron drops what PHP prints: an error that ends a run is a problem like any other. - $what = $copy ? 'Replication of '.$job['dataset'].' to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']) : 'Scheduled snapshots'; + $what = $copy ? 'Replication of '.$job['dataset'].' to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']) : (zfsmScheduleSnapshotsOnly($job) ? 'Scheduled snapshots' : 'Scheduled copies'); zfsmLog($what.' failed: '.zfsmOneLine($error->getMessage())); - $notify($copy ? 'ZFS Replication' : 'ZFS Snapshot', $what.' failed: '.zfsmSafeText(zfsmOneLine($error->getMessage())), 'warning'); + $notify(zfsmScheduleSnapshotsOnly($job) ? 'ZFS Snapshot' : 'ZFS Replication', $what.' failed: '.zfsmSafeText(zfsmOneLine($error->getMessage())), 'warning'); return 1; } } @@ -380,8 +386,8 @@ function zfsmSchedulesHtml($cfg) { foreach ($read['valid'] as $job) { $failures = zfsmScheduleProblems($job); $clash = zfsmScheduleClash($job, $read['valid'], $cfg, time(), false); - $text = zfsmScheduleSnapshotsOnly($job) ? zfsmScheduleSnapshotText($job) - : $job['dataset'].' to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']).', '.zfsmScheduleWhen($job).', '.zfsmScheduleKeepText($job); + $text = zfsmScheduleSnapshotsOnly($job) ? zfsmScheduleSnapshotText($job) : (zfsmScheduleCopiesUnder($job) ? zfsmScheduleCopiesText($job) + : $job['dataset'].' to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']).', '.zfsmScheduleWhen($job).', '.zfsmScheduleKeepText($job)); // The default changes and turns off in its own row above; a dataset's goes here too, as one whose dataset is gone can't be opened. $html .= ''.zfsmHtml($text) .($job['dataset'] !== '' ? ' ' : '') diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php new file mode 100644 index 00000000..1a141421 --- /dev/null +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -0,0 +1,137 @@ + false, 'why' => 'Nothing was saved: '.$wrong, 'said' => ''); + // One zfsmScheduleCoverRead: Copy Under must be there, and the answer counts what a run copies. + $listed = $remove ? null : zfsmScheduleCoverRead($cfg, $wait); + $not = $remove ? '' : zfsmScheduleCopiesUnderNot($under, $listed); + if ($not !== '') return array('saved' => false, 'why' => 'Nothing was saved: '.$not, 'said' => ''); + $after = array(); + $changed = zfsmScheduleSnapshotChange(function($all) use (&$job, $remove, $cfg) { return zfsmScheduleSnapshotPut($all, $job, $remove, $cfg); }, $after); + if (!$changed['saved']) return $changed + array('said' => ''); + if ($remove) return $changed + array('said' => 'Replicate Every Dataset is off; the copies it made and their snapshots stay'); + list($pools, $unread) = zfsmScheduleCovered($listed, $job, $after); + $count = $unread !== '' && !$pools ? 'each dataset' : 'each of '.array_sum(array_map('count', $pools)).' dataset(s)'; + return $changed + array('said' => 'Saved: a copy of '.$count.' under '.$under.' '.zfsmScheduleWhen($job).'; it '.zfsmScheduleKeepText($job) + .($unread !== '' ? '; not every dataset could be read: '.$unread : '')); +} + +// Cron's run of copies under a dataset: each dataset it covers, parents first, sent to its copy as a copy's schedule sends it +// (zfsmRunReplicate, with its Keep); one notification for the run. 0, or 1 after a problem. +function zfsmScheduleCopiesRun($job, $jobs, $cfg, $now, $notify, $wait = ZFSM_SCHEDULE_WAIT) { + return zfsmScheduleLockedRun($job, 'ZFS Replication', 'Scheduled copies not made: ', $notify, function() use ($job, $jobs, $cfg, $now, $wait) { + return zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait); + }); +} + +// What a run copied, and whether it had a problem: null when zfs read every pool and none had a dataset to copy. Its waits for other +// replications of a dataset share $wait, so the run waits that long at most in all. +function zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait) { + $under = substr($job['dest'], 0, -1); + $listed = zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT); + list($pools, $unread) = zfsmScheduleCovered($listed, $job, $jobs); + if (!$pools) return array('Scheduled copies not made: '.($unread !== '' ? 'the datasets could not be read: '.$unread : 'no pool has a dataset it copies'), $unread !== '' ? true : null); + $not = zfsmScheduleCopiesUnderNot($under, $listed); + if ($not !== '') return array('Scheduled copies not made: '.$not, true); + $datasets = array_merge(...array_values($pools)); + // By name: a dataset's parents come before it. + sort($datasets, SORT_STRING); + $here = array_fill_keys(array_keys(zfsmScheduleListed($listed)[0]), true); + $covered = array_fill_keys($datasets, true); + $name = zfsmNewSnapshotName($cfg, $now); + $until = time() + $wait; + $made = 0; + $problems = $waiting = array(); + foreach ($datasets as $zdataset) { + $zdest = zfsmScheduleCopiesDest($job, $zdataset); + $place = zfsmScheduleCopiesPlace($job, $zdest, $covered, $here); + if ($place === null) $waiting[] = $zdataset; + if ($place !== '') { + if ($place !== null) $problems[] = $place; + continue; + } + $said = array('', 'normal'); + $tell = function($title, $text, $type) use (&$said) { $said = array(rtrim($text, '.'), $type); }; + $code = zfsmRunReplicate($zdataset, $name, $zdest, null, true, '', array('notify' => $tell, 'wait' => max(0, $until - time()), 'tag' => zfsmKeepTag('', $job['dest'])), $job['keep']); + if ($code === 0) { + $made++; + $here[$zdest] = true; + } + if ($code !== 0 || $said[1] !== 'normal') $problems[] = $said[0]; + } + return array(zfsmScheduleCopiesSaid($under, $made.' of '.count($datasets), $waiting, $unread, $problems), $waiting || $unread !== '' || $problems); +} + +// A run's line: how many it copied, those waiting for their parent's copy, the datasets not read, the first problems. +function zfsmScheduleCopiesSaid($under, $made, $waiting, $unread, $problems) { + return 'Scheduled copies under '.$under.': '.$made.' dataset(s) copied' + // Before the problems, which the log's cut can drop (zfsmSafeText); in brackets and commas, where diagnostics end a name. + .($waiting ? '. Not copied, as the copy of a dataset above it is missing ('.zfsmKeepFirst($waiting, ', ').')' : '') + .($unread !== '' ? '. Not every dataset could be read: '.$unread : '').($problems ? '. '.zfsmKeepFirst($problems) : ''); +} + +// Whether the datasets above $zdest are there for its copy: each missing one below $job's dest is made a holder +// (zfsmReplicaHolderMake), but where a dataset this run covers gets its copy. '' when they are, null to wait for that copy, else +// why one could not be made. $here gains each holder made. +function zfsmScheduleCopiesPlace($job, $zdest, $covered, &$here) { + $level = substr($job['dest'], 0, -1); + foreach (array_slice(explode('/', substr($zdest, strlen($job['dest']))), 0, -1) as $part) { + $level .= '/'.$part; + if (isset($here[$level])) continue; + if (isset($covered[substr($level, strlen($job['dest']))])) return null; + $why = zfsmReplicaHolderMake($level); + if ($why !== '') return $why; + $here[$level] = true; + } + return ''; +} diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index d1211cf9..8a266801 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -1,6 +1,7 @@ 'off', 'minute' => 0, 'hour' => 3, 'days' => range(0, 6), 'keep' => 0); - $job = zfsmScheduleFollowed('', zfsmScheduleRead()['valid'] ?? array()); +// What Settings' row of the default of $kind shows: it (an hourly one with Off's Starts At, which it doesn't use), or Off when there +// is none or schedules.json can't be read (the list under it says so). +function zfsmScheduleShown($kind = 'zfsmScheduleSnapshotsOnly') { + $off = array('every' => 'off', 'dest' => '', 'minute' => 0, 'hour' => 3, 'days' => range(0, 6), 'keep' => 0); + $job = zfsmScheduleFollowed('', zfsmScheduleRead()['valid'] ?? array(), $kind); if ($job === null) return $off; return $job['every'] === 'hourly' ? array('minute' => $off['minute'], 'hour' => $off['hour']) + $job : $job; } @@ -109,13 +115,20 @@ function zfsmScheduleSkipped($zdataset, $row) { return $row[1] === 'legacy' ? $zdataset.' is mounted legacy, as Docker image layers are' : ''; } -// From $listed: by pool, each dataset that follows $job (zfsmScheduleFollowed in $jobs) and a run can take (zfsmScheduleSkipped); and -// why some weren't read ('' when all were). +// Why no run of $job takes $zdataset, listed as $row: zfsmScheduleSkipped's reasons, and for copies under a dataset +// zfsmScheduleCopiesNot's; '' when one can. +function zfsmScheduleLeftOut($job, $jobs, $zdataset, $row) { + $why = zfsmScheduleSkipped($zdataset, $row); + return $why === '' && zfsmScheduleCopiesUnder($job) ? zfsmScheduleCopiesNot($job, $jobs, $zdataset) : $why; +} + +// From $listed: by pool, each dataset that follows $job (zfsmScheduleFollowed of its kind in $jobs) and a run can take +// (zfsmScheduleLeftOut); and why some weren't read ('' when all were). function zfsmScheduleCovered($listed, $job, $jobs) { list($rows, $errors) = zfsmScheduleListed($listed); $pools = array(); foreach ($rows as $zdataset => $row) { - if (zfsmScheduleSkipped($zdataset, $row) === '' && (zfsmScheduleFollowed($zdataset, $jobs)['id'] ?? null) === $job['id']) $pools[explode('/', $zdataset)[0]][] = $zdataset; + if (zfsmScheduleLeftOut($job, $jobs, $zdataset, $row) === '' && (zfsmScheduleFollowed($zdataset, $jobs, zfsmScheduleKindOf($job))['id'] ?? null) === $job['id']) $pools[explode('/', $zdataset)[0]][] = $zdataset; } // zfs lists what it can read, and fails for the rest: a dataset's own minds errors naming none, its datasets or one above. if (($listed['code'] ?? 1) === 0) return array($pools, ''); @@ -168,7 +181,8 @@ function zfsmScheduleSnapshotSave($zdataset, $every, $time, $days, $keep, $cfg, return $changed + array('said' => zfsmScheduleSnapshotSaid($job, $remove, $after, $listed)); } -// zfsmScheduleChange for snapshot records, $after as saved: one that goes or is None (it never runs) loses its count of runs with a problem. +// zfsmScheduleChange for snapshot records and copies under a dataset, $after as saved: one that goes or is None (it never runs) +// loses its count of runs with a problem. function zfsmScheduleSnapshotChange($change, &$after = array()) { $stale = array(); $changed = zfsmScheduleChange(function($before) use ($change, &$after, &$stale) { @@ -210,10 +224,10 @@ function zfsmScheduleSnapshotFollow($answer, $from, $to) { return $answer; } -// $jobs with $job in place of its dataset's snapshot schedule (none with $remove), keeping that one's id and an hourly one's minute; -// or why it can't be. +// $jobs with $job in place of its dataset's schedule of its kind (none with $remove), keeping that one's id and an hourly one's +// minute; or why it can't be. function zfsmScheduleSnapshotPut($jobs, &$job, $remove, $cfg) { - $old = zfsmScheduleOwn($jobs, $job['dataset']); + $old = zfsmScheduleOwn($jobs, $job['dataset'], zfsmScheduleKindOf($job)); $jobs = array_diff_key($jobs, $old); if ($remove) return $jobs; $was = reset($old); @@ -265,15 +279,22 @@ function zfsmScheduleSnapshotKeep($pool, $taken, $keep) { // Cron's run of a snapshot schedule ($jobs: all of them, which share the datasets out): a snapshot of every dataset it covers, then its // Keep; logged, and given to $notify. 0, or 1 after a problem. function zfsmScheduleSnapshotRun($job, $jobs, $cfg, $now, $notify, $wait = ZFSM_SCHEDULE_COVER_WAIT) { - // One at a time: a run still going at the next one's time (a long Keep, a hung pool) stops that one, so runs don't pile up. + return zfsmScheduleLockedRun($job, 'ZFS Snapshot', 'Scheduled snapshots not taken: ', $notify, function() use ($job, $jobs, $cfg, $now, $wait) { + return zfsmScheduleSnapshotsTaken($job, $jobs, $cfg, $now, $wait); + }); +} + +// $run of $job (array(text, problem) as zfsmScheduleSnapshotTell takes them) under the job's lock; $not starts what a run that can't +// start says. One at a time: a run still going at the next one's time (a long Keep, a hung pool) stops that one, so runs don't pile up. +function zfsmScheduleLockedRun($job, $title, $not, $notify, $run) { $lock = zfsmJobLock(zfsmJobDir('schedule').'/'.$job['id'].'.lock'); - if (!is_resource($lock)) return zfsmScheduleSnapshotTell($job, 'Scheduled snapshots not taken: '.($lock === null ? 'the lock file could not be opened' : 'the run before is still going'), true, $notify); + if (!is_resource($lock)) return zfsmScheduleSnapshotTell($job, $not.($lock === null ? 'the lock file could not be opened' : 'the run before is still going'), true, $notify, $title); try { - list($text, $problem) = zfsmScheduleSnapshotsTaken($job, $jobs, $cfg, $now, $wait); + list($text, $problem) = $run(); } finally { zfsmJobUnlock($lock); } - return zfsmScheduleSnapshotTell($job, $text, $problem, $notify); + return zfsmScheduleSnapshotTell($job, $text, $problem, $notify, $title); } // What a run took and kept, and whether it had a problem: null when zfs read every pool and the default had no dataset to take. @@ -313,9 +334,10 @@ function zfsmScheduleSnapshotNone($job, $listed, $unread) { return array('Scheduled snapshots of '.$job['dataset'].' not taken: '.zfsmScheduleSnapshotNot($job['dataset'], $listed).'; remove its schedule in Settings', true); } -// Logs a run's $text and has $notify tell it; 1 for a problem. A null $problem (nothing to take) is only logged: no run that works again. -function zfsmScheduleSnapshotTell($job, $text, $problem, $notify) { +// Logs a run's $text and has $notify tell it under $title; 1 for a problem. A null $problem (nothing to take) is only logged: no run +// that works again. +function zfsmScheduleSnapshotTell($job, $text, $problem, $notify, $title = 'ZFS Snapshot') { zfsmLog('Schedule '.$job['id'].': '.$text, $problem ? LOG_WARNING : LOG_NOTICE); - if ($problem !== null) $notify('ZFS Snapshot', zfsmSafeText($text).'.', $problem ? 'warning' : 'normal'); + if ($problem !== null) $notify($title, zfsmSafeText($text).'.', $problem ? 'warning' : 'normal'); return $problem ? 1 : 0; } From 1818729b2020534f473bf29f0ffec8b7f5e81bf6 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 07:44:28 +0800 Subject: [PATCH 2/8] fix(schedules): Replicate Every Dataset waits for a copy's own schedule and leaves out what it cannot copy - No holder where a copy's own schedule puts its copy: the datasets below wait for it - An error in one dataset's replication ends only that one - Only waits use a run's wait budget, not its sends - Leaves out datasets whose snapshot names could not be read, and names too long for the bookmark - zfsmScheduleCovers and zfsmScheduleDaysHtml each own one rule; the gone hint is the caller's - Help: a new Copy Under leaves the old copies; a run due while one still goes counts as a problem --- tests/schedule_copies_test.php | 60 ++++++++++++++-- tests/unraid_compat_check.php | 8 ++- zfs.master/ZFSMasterSettings.page | 12 ++-- zfs.master/include/ZFSMDiagPlugin.php | 2 +- zfs.master/include/ZFSMReplicate.php | 16 +++-- zfs.master/include/ZFSMSchedule.php | 21 ++++-- zfs.master/include/ZFSMScheduleCopies.php | 77 +++++++++++++++------ zfs.master/include/ZFSMScheduleSnapshot.php | 30 ++++---- 8 files changed, 168 insertions(+), 58 deletions(-) diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 443327a4..e53781e5 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -9,6 +9,14 @@ }); require __DIR__.'/fixtures/fake_bin_guard.php'; + +// Unraid's nchan publish, which a run calls as it sends: 'zfsm_publish' 'throw' throws once, a number sleeps that long once. +function publish($channel, $message) { + $then = $GLOBALS['zfsm_publish'] ?? null; + $GLOBALS['zfsm_publish'] = null; + if ($then === 'throw') throw new RuntimeException('nchan is down'); + if (is_int($then)) sleep($then); +} require_once __DIR__.'/../zfs.master/backend/ZFSMOperations.php'; function same($expected, $actual, $message) { @@ -81,8 +89,11 @@ function same($expected, $actual, $message) { same(array(array('tank' => array('tank', 'tank/data', 'tank/docker', 'tank/san/kid', 'tank/vm'), 'usb' => array('usb')), ''), zfsmScheduleCovered($listed(), $every, array($every)), 'it covers each dataset but those left out'); same(array('tank', 'tank/docker', 'tank/san/kid', 'tank/vm'), zfsmScheduleCovered($listed(), $every, array($every, array('dest' => 'backup/tank/data') + $copy))[0]['tank'], 'not one whose own copy is that copy'); same(array('tank', 'tank/data', 'tank/docker', 'tank/san/kid', 'tank/vm'), zfsmScheduleCovered($listed(), $every, array($every, array('dest' => 'backup/data') + $copy))[0]['tank'], 'but one whose own copy is elsewhere'); -same(array('backup/old is in pool backup, where its copy would go', 'tank/data has a schedule of its own for that copy', ''), - array(zfsmScheduleCopiesNot($every, array(), 'backup/old'), zfsmScheduleCopiesNot($every, array(array('dest' => 'backup/tank/data') + $copy), 'tank/data'), zfsmScheduleCopiesNot($every, array(), 'tank/data')), 'why one is left out'); +$not = function($zdataset, $jobs = array(), $row = array('off', '/mnt/x', false)) use ($every, $name) { return zfsmScheduleCopiesNot($every, $jobs, $zdataset, $row, $name(1800000000)); }; +same(array('backup/old is in pool backup, where its copy would go', 'tank/data has a schedule of its own for that copy', '', 'the snapshot names of tank/data could not be read, so it may be left to Sanoid or Syncoid'), + array($not('backup/old'), $not('tank/data', array(array('dest' => 'backup/tank/data') + $copy)), $not('tank/data'), $not('tank/data', array(), array('off', '/mnt/tank/data', null))), 'why one is left out'); +// The bookmark that keeps a copy incremental names the dataset and its copy (zfsmReplicaBookmarkFits): under backup, 71 fit. +same(array('', 'the names of tank/'.str_repeat('a', 67).' and its copy are too long for the bookmark a copy keeps'), array($not('tank/'.str_repeat('a', 66)), $not('tank/'.str_repeat('a', 67))), 'one too long'); // Save: Copy Under must be a filesystem here; what is wrong is said, and nothing saved. $wrong = array( @@ -162,15 +173,17 @@ function same($expected, $actual, $message) { same(array(), $snaps('tank/san'), 'the snapshot it took for it goes'); same(array("backup/tank/san/kid@$o"), array_keys($snaps('backup/tank/san/kid')), 'the one below is copied'); -// Other replications of tank/data and tank/docker still running: the run's waits share its budget, so the second has none left. +// Other replications of tank/data and tank/docker still running, after tank's send outlasted the budget: only waits use it, and +// they share it, so the second has none left. $fresh(); $reset(); @unlink(zfsmScheduleStateFile($id)); $locks = array_map(function($dataset) { return zfsmJobLock(zfsmJobLockFile($dataset, 'replicate')); }, array('tank/data', 'tank/docker')); $started = time(); +$GLOBALS['zfsm_publish'] = 3; same(1, zfsmScheduleRun($id, $cfg, $t + 14400, 2), 'a run that waits'); array_map('zfsmJobUnlock', $locks); -same(true, time() - $started < 5, 'in all, about its budget'); +same(true, time() - $started < 8, 'in all, its budget and the slow send'); same(1, preg_match('/^warning: Scheduled copies under backup: 4 of 6 dataset\(s\) copied\. Replication of tank\/data to backup\/tank\/data not started: another replication of this dataset was still running after this run waited 1 minute\(s\) for it; Replication of tank\/docker to backup\/tank\/docker not started: another replication of this dataset was still running, and the run had no time left to wait for it/', $read('notify-detail.log')[0] ?? ''), 'each says so'); // A run while the one before still goes: none made, notified. @@ -181,6 +194,29 @@ function same($expected, $actual, $message) { zfsmJobUnlock($held); same(array(array('warning: ZFS Replication'), 1), array($read('notify.log'), preg_match('/^warning: Scheduled copies not made: the run before is still going/', $read('notify-detail.log')[0] ?? '')), 'notified'); +// An error in one dataset's replication ends only that one: the datasets below it wait, usb is copied. +$fresh(); +$reset(); +@unlink(zfsmScheduleStateFile($id)); +$GLOBALS['zfsm_publish'] = 'throw'; +same(1, zfsmScheduleRun($id, $cfg, $t + 19800), 'a run that an error ends for one dataset'); +same(array('backup/usb'), array_values(preg_grep('~^backup/(?!old$)~', $datasets())), 'the others go on'); +same(1, preg_match('/^warning: Scheduled copies under backup: 1 of 6 dataset\(s\) copied\. Not copied, as the copy of a dataset above it is missing \(tank\/data, tank\/docker, tank\/san\/kid, and 1 more\)\. Replication of tank to backup\/tank failed: nchan is down\. Later problems/', $read('notify-detail.log')[0] ?? ''), 'naming it'); + +// A dataset gone before its turn: said, with no schedule of its own to remove. +$left = 2; +same(array(1, array('Replication of tank/gone to backup/tank/gone not started: it no longer exists', 'warning')), zfsmScheduleCopiesOne($every, 'tank/gone', 'backup/tank/gone', $name($t), $left), 'one gone before its turn'); + +// The snapshot names not read: which datasets are Sanoid's isn't known, so none is copied, and no snapshot taken. +$fresh(); +$reset(); +@unlink(zfsmScheduleStateFile($id)); +putenv('ZFSM_TEST_NAMES_FAIL=1'); +same(1, zfsmScheduleRun($id, $cfg, $t + 20400), 'a run without the names'); +putenv('ZFSM_TEST_NAMES_FAIL'); +same(array(array('backup', 'backup/old'), array()), array(array_values(preg_grep('~^backup~', $datasets())), preg_grep('/^(snapshot|send|create) /', $read('calls.log'))), 'nothing taken or copied'); +same(1, preg_match('/^warning: Scheduled copies not made: the datasets could not be read: cannot iterate filesystems: Input\/output error/', $read('notify-detail.log')[0] ?? ''), 'saying why'); + // Copy Under gone: none made; a later problem, so only logged. $pool('datasets', array_values(array_filter($read('pool/datasets'), function($line) { return strpos($line, 'backup') !== 0; }))); $reset(); @@ -188,4 +224,20 @@ function same($expected, $actual, $message) { same(array(), $read('notify.log'), 'a later problem is only logged'); same(1, $logged('/ Schedule '.$id.': Scheduled copies not made: backup does not exist on this server$/'), 'saying why'); +// A copy's own schedule makes tank/data's copy, not made yet: no holder goes there, which would refuse it; the one below waits. +$pool('datasets', array("tank\tfilesystem", "tank/data\tfilesystem", "tank/data/sub\tfilesystem", "backup\tfilesystem")); +$pool('marks', array()); +$pool('props', array()); +$own = array('id' => 'bbbb0000', 'dataset' => 'tank/data', 'dest' => 'backup/tank/data', 'minute' => 0) + $every; +file_put_contents(zfsmScheduleFile(), json_encode(array(array('keep' => 1) + $every, $own))); +$reset(); +@unlink(zfsmScheduleStateFile($id)); +same(1, zfsmScheduleRun($id, $cfg, $t + 25200), 'a run with a copy to come'); +same(array(array('backup/tank'), array()), array(array_values(preg_grep('~^backup/~', $datasets())), preg_grep('/^create /', $read('calls.log'))), 'no holder where it goes'); +same(1, preg_match('/^warning: Scheduled copies under backup: 1 of 2 dataset\(s\) copied\. Not copied, as the copy of a dataset above it is missing \(tank\/data\/sub\)\. Later problems/', $read('notify-detail.log')[0] ?? ''), 'it waits'); +same(array(0, true), array(zfsmScheduleRun('bbbb0000', $cfg, $t + 25260), in_array('backup/tank/data', $datasets(), true)), 'the copy lands'); +$reset(); +same(0, zfsmScheduleRun($id, $cfg, $t + 28800), 'the next run works'); +same(true, in_array('backup/tank/data/sub', $datasets(), true), 'and copies the one below'); + echo "schedule copies tests passed\n"; diff --git a/tests/unraid_compat_check.php b/tests/unraid_compat_check.php index e20fe24c..e9291b6f 100644 --- a/tests/unraid_compat_check.php +++ b/tests/unraid_compat_check.php @@ -269,9 +269,15 @@ function popup_styles($html, $docroot) { if (!$row_has('Snapshot Time', '', 'Snapshots To Keep' => 'id="zfsm-snapsched-keep"') as $label => $control) { + foreach (array('Snapshot Every Dataset' => '', 'Copy Under' => 'id="zfsm-copysched-under"') as $label => $control) { if (!$row_has($label, $control)) { fwrite(STDERR, "settings page lost the $label row\n"); exit(1); } } + // Each Every Dataset row's Days, all seven ticked while it is Off (zfsmScheduleShown). + foreach (array('zfsm-snapsched', 'zfsm-copysched') as $prefix) { + if (!$row_has('Days', '') || preg_match_all('# (?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)#', $rendered) !== 7) { + fwrite(STDERR, "settings page lost the Days of $prefix, or a day is not ticked\n"); exit(1); + } + } foreach (array('Pool Health Alerts' => 'alert_interval', 'Stale Snapshot Alerts' => 'alert_stale', 'Size Units' => 'size_units') as $label => $name) { if (!$row_has($label, '> +: _(Starts At)_: : @@ -346,11 +346,11 @@ _(Replicate Every Dataset)_: > Copies every dataset in every pool into the dataset you name under **Copy Under**, on this server, each copy named for its dataset: with `backup`, `cache/appdata` is copied to `backup/cache/appdata`. It runs every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. Save changes it, and the answer says how many datasets it copies. Off stops it; the copies and the snapshots it made stay. > > - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full. The copies are read-only and not mounted. -> - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy. -> - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. When the copy of the dataset above one is missing because it failed, that one waits for the next run. -> - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. +> - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy, one whose name is too long for the bookmark its copy keeps (over about 70 characters), and, when a run can't read the snapshot names, those it couldn't read: they may be left to Sanoid or Syncoid. +> - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. When the copy of the dataset above one is missing because it failed, or its own copy schedule has not made it yet, that one waits for the next run. +> - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. -> - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. +> - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. So does a run that falls due while the one before still goes, as a first run can. > - Each run sends one notification at most: what it copied and its first problems. The first run with a problem sends it; later ones are logged in `/var/log/zfs.master.log` until a run works again, and that run's notification says so. The log has each dataset's replication. :end @@ -358,7 +358,7 @@ _(Copy Under)_: : _(Days)_: -: +: _(Starts At)_: : diff --git a/zfs.master/include/ZFSMDiagPlugin.php b/zfs.master/include/ZFSMDiagPlugin.php index 76d1d4f5..8b4f49b1 100644 --- a/zfs.master/include/ZFSMDiagPlugin.php +++ b/zfs.master/include/ZFSMDiagPlugin.php @@ -411,7 +411,7 @@ function zfsmDiagPublicHalf($dir, $private, $public) { // How many datasets each snapshot schedule and copies under a dataset that runs covers now, by id, from reads under a limit of their // own so a hung pool costs only this line; '' with none. function zfsmDiagScheduleCovers($jobs, $cfg) { - $runs = array_filter($jobs, function($job) { return (zfsmScheduleSnapshotsOnly($job) || zfsmScheduleCopiesUnder($job)) && zfsmScheduleRepeats($job['every']); }); + $runs = array_filter($jobs, function($job) { return zfsmScheduleCovers($job) && zfsmScheduleRepeats($job['every']); }); if (!$runs) return ''; $listed = zfsmScheduleCoverRead($cfg, 5, array(), function($wait, $command, $args) { return zfsmDiagRead($command, $args, $wait); }); $counts = array(); diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index 672f017f..cf2e81fd 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -138,7 +138,7 @@ function zfsmReplicaMark($zdataset, $plan) { // $take: the run takes $snapshot first, under its lock, and removes it again unless a copy holds it or a stopped run may need it. // $started runs once the plan holds, before zfs send; until then the reason a run doesn't start also goes to the starting page. // $schedule, for a scheduled run (zfsmScheduleRun): its 'notify'(title, text, importance), the seconds to 'wait' out another run, and no full send to a copy that is gone; -// 'tag' for a schedule of many datasets (zfsmScheduleCopiesRun), which names its holds and Keep instead of the copy's. +// optional: 'waited'(seconds) told how long it waited, what to do when the dataset is 'gone', and a 'tag' (zfsmKeepTag) for its holds and Keep. // $keep, for any run of a schedule: the snapshot it takes is tagged for the copy (zfsmKeepTag), and once it lands only those of // them $keep keeps stay (0: all, zfsmKeepPrune). function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = false, $server = '', $schedule = null, $keep = null) { @@ -158,9 +158,8 @@ function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = return $not_started($why); }; $lock_file = zfsmJobLockFile($zdataset, 'replicate'); - if ($lock_file === null) return $not_started(zfsmDatasetMissing($zdataset) === true ? 'it no longer exists'.($schedule !== null && !isset($schedule['tag']) ? '; remove its schedule in Settings' : '') : 'its guid could not be read'); - $lock = zfsmJobLock($lock_file); - for ($until = time() + ($schedule['wait'] ?? 0); $lock === false && time() < $until; $lock = zfsmJobLock($lock_file)) sleep(max(1, min(10, $until - time()))); + if ($lock_file === null) return $not_started(zfsmDatasetMissing($zdataset) === true ? 'it no longer exists'.(isset($schedule['gone']) ? '; '.$schedule['gone'] : '') : 'its guid could not be read'); + $lock = zfsmReplicateLock($lock_file, $schedule); if ($lock === null) return $not_started('its lock file could not be opened'); // No page starts a scheduled run again: one that waited in vain is a problem. $waited = (int)ceil(($schedule['wait'] ?? 0) / 60); @@ -194,6 +193,15 @@ function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = } } +// zfsmJobLock of $lock_file, waiting out another run up to $schedule's 'wait' seconds; its 'waited' is told how long. +function zfsmReplicateLock($lock_file, $schedule) { + $lock = zfsmJobLock($lock_file); + $since = time(); + for ($until = $since + ($schedule['wait'] ?? 0); $lock === false && time() < $until; $lock = zfsmJobLock($lock_file)) sleep(max(1, min(10, $until - time()))); + if (isset($schedule['waited'])) $schedule['waited'](time() - $since); + return $lock; +} + // One plan of zfsmRunReplicate's sent, with its log lines and the row's progress: array(0 when it landed, its notification's text and type). // $tag: the copy's (zfsmKeepTag), whose holds each run puts on before its send and a landed one takes off. function zfsmReplicateSent($plan, $zdataset, $snapshot, $take, $target, $tag) { diff --git a/zfs.master/include/ZFSMSchedule.php b/zfs.master/include/ZFSMSchedule.php index 343a5fff..38ab52e4 100644 --- a/zfs.master/include/ZFSMSchedule.php +++ b/zfs.master/include/ZFSMSchedule.php @@ -135,14 +135,12 @@ function zfsmScheduleFreeMinute($jobs, $job, $prefer = null) { return $free ? $free[array_rand($free)] : random_int(0, 59); } -// Whether $a and $b can snapshot one dataset (so not in one minute or name): both of it, or one that covers datasets below its own -// (snapshots, copies under a dataset) of one above it or the default. Each dataset follows one of each kind (zfsmScheduleFollowed); -// None takes none. +// Whether $a and $b can snapshot one dataset (so not in one minute or name): both of it, or one that covers (zfsmScheduleCovers) of +// one above it or the default. A dataset follows one of each kind (zfsmScheduleFollowed); None takes none. function zfsmScheduleShare($a, $b) { if (!zfsmScheduleRepeats($a['every']) || !zfsmScheduleRepeats($b['every'])) return false; - $covers = function($job) { return zfsmScheduleSnapshotsOnly($job) || zfsmScheduleCopiesUnder($job); }; - if ($covers($a) && $covers($b) && zfsmScheduleKindOf($a) === zfsmScheduleKindOf($b)) return false; - $in = function($job, $other) use ($covers) { return $covers($job) && ($job['dataset'] === '' || zfsmUnderAny($other['dataset'], array($job['dataset']))); }; + if (zfsmScheduleCovers($a) && zfsmScheduleCovers($b) && zfsmScheduleKindOf($a) === zfsmScheduleKindOf($b)) return false; + $in = function($job, $other) { return zfsmScheduleCovers($job) && ($job['dataset'] === '' || zfsmUnderAny($other['dataset'], array($job['dataset']))); }; return $a['dataset'] === $b['dataset'] || $in($a, $b) || $in($b, $a); } @@ -368,7 +366,7 @@ function zfsmScheduleRun($id, $cfg, $now, $wait = ZFSM_SCHEDULE_WAIT) { try { if (zfsmScheduleCopiesUnder($job)) return zfsmScheduleCopiesRun($job, $read['valid'], $cfg, $now, $notify, $wait); if (!$copy) return zfsmScheduleSnapshotRun($job, $read['valid'], $cfg, $now, $notify); - return zfsmRunReplicate($job['dataset'], zfsmNewSnapshotName($cfg, $now), $job['dest'], null, true, $job['server'], array('notify' => $notify, 'wait' => $wait), $job['keep']); + return zfsmRunReplicate($job['dataset'], zfsmNewSnapshotName($cfg, $now), $job['dest'], null, true, $job['server'], array('notify' => $notify, 'wait' => $wait, 'gone' => 'remove its schedule in Settings'), $job['keep']); } catch (Throwable $error) { // Cron drops what PHP prints: an error that ends a run is a problem like any other. $what = $copy ? 'Replication of '.$job['dataset'].' to '.zfsmReplicaTargetWhere(array('server' => $job['server']), $job['dest']) : (zfsmScheduleSnapshotsOnly($job) ? 'Scheduled snapshots' : 'Scheduled copies'); @@ -379,6 +377,15 @@ function zfsmScheduleRun($id, $cfg, $now, $wait = ZFSM_SCHEDULE_WAIT) { } // Settings' list; $cfg for Snapshots Pattern, which can change after a schedule was saved. +// Settings' Days boxes of the row whose ids start $prefix, $days ticked. +function zfsmScheduleDaysHtml($prefix, $days) { + $html = ''; + foreach (array(1, 2, 3, 4, 5, 6, 0) as $day) { + $html .= ''; + } + return ''.$html.''; +} + function zfsmSchedulesHtml($cfg) { $read = zfsmScheduleRead(); if ($read === null) return ''.zfsmHtml(zfsmScheduleUnreadable()).''; diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index 1a141421..738038c7 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -1,7 +1,6 @@ $job['server']), $dest), $name)) return 'the names of '.$zdataset.' and its copy are too long for the bookmark a copy keeps'; foreach ($jobs as $other) { if ($other['dataset'] === $zdataset && $other['dest'] === $dest && $other['server'] === $job['server']) return $zdataset.' has a schedule of its own for that copy'; } @@ -59,11 +60,15 @@ function zfsmScheduleCopiesSave($under, $every, $time, $days, $keep, $cfg, $wait $after = array(); $changed = zfsmScheduleSnapshotChange(function($all) use (&$job, $remove, $cfg) { return zfsmScheduleSnapshotPut($all, $job, $remove, $cfg); }, $after); if (!$changed['saved']) return $changed + array('said' => ''); - if ($remove) return $changed + array('said' => 'Replicate Every Dataset is off; the copies it made and their snapshots stay'); + return $changed + array('said' => $remove ? 'Replicate Every Dataset is off; the copies it made and their snapshots stay' : zfsmScheduleCopiesSaved($job, $listed, $after)); +} + +// Save's answer for $job: what a run copies, counted from $listed (zfsmScheduleCoverRead) with the schedules saved, $after. +function zfsmScheduleCopiesSaved($job, $listed, $after) { list($pools, $unread) = zfsmScheduleCovered($listed, $job, $after); $count = $unread !== '' && !$pools ? 'each dataset' : 'each of '.array_sum(array_map('count', $pools)).' dataset(s)'; - return $changed + array('said' => 'Saved: a copy of '.$count.' under '.$under.' '.zfsmScheduleWhen($job).'; it '.zfsmScheduleKeepText($job) - .($unread !== '' ? '; not every dataset could be read: '.$unread : '')); + return 'Saved: a copy of '.$count.' under '.substr($job['dest'], 0, -1).' '.zfsmScheduleWhen($job).'; it '.zfsmScheduleKeepText($job) + .($unread !== '' ? '; not every dataset could be read: '.$unread : ''); } // Cron's run of copies under a dataset: each dataset it covers, parents first, sent to its copy as a copy's schedule sends it @@ -80,29 +85,25 @@ function zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait) { $under = substr($job['dest'], 0, -1); $listed = zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT); list($pools, $unread) = zfsmScheduleCovered($listed, $job, $jobs); - if (!$pools) return array('Scheduled copies not made: '.($unread !== '' ? 'the datasets could not be read: '.$unread : 'no pool has a dataset it copies'), $unread !== '' ? true : null); - $not = zfsmScheduleCopiesUnderNot($under, $listed); - if ($not !== '') return array('Scheduled copies not made: '.$not, true); + $none = zfsmScheduleCopiesNone($under, $listed, $pools, $unread); + if ($none !== null) return $none; $datasets = array_merge(...array_values($pools)); // By name: a dataset's parents come before it. sort($datasets, SORT_STRING); $here = array_fill_keys(array_keys(zfsmScheduleListed($listed)[0]), true); - $covered = array_fill_keys($datasets, true); + $coming = zfsmScheduleCopiesComing($job, $jobs, $datasets); $name = zfsmNewSnapshotName($cfg, $now); - $until = time() + $wait; $made = 0; $problems = $waiting = array(); foreach ($datasets as $zdataset) { $zdest = zfsmScheduleCopiesDest($job, $zdataset); - $place = zfsmScheduleCopiesPlace($job, $zdest, $covered, $here); + $place = zfsmScheduleCopiesPlace($job, $zdest, $coming, $here); if ($place === null) $waiting[] = $zdataset; if ($place !== '') { if ($place !== null) $problems[] = $place; continue; } - $said = array('', 'normal'); - $tell = function($title, $text, $type) use (&$said) { $said = array(rtrim($text, '.'), $type); }; - $code = zfsmRunReplicate($zdataset, $name, $zdest, null, true, '', array('notify' => $tell, 'wait' => max(0, $until - time()), 'tag' => zfsmKeepTag('', $job['dest'])), $job['keep']); + list($code, $said) = zfsmScheduleCopiesOne($job, $zdataset, $zdest, $name, $wait); if ($code === 0) { $made++; $here[$zdest] = true; @@ -112,6 +113,37 @@ function zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait) { return array(zfsmScheduleCopiesSaid($under, $made.' of '.count($datasets), $waiting, $unread, $problems), $waiting || $unread !== '' || $problems); } +// What a run says when it copies none, as zfsmScheduleCopiesMade answers; null when it can copy. +function zfsmScheduleCopiesNone($under, $listed, $pools, $unread) { + if (!$pools) return array('Scheduled copies not made: '.($unread !== '' ? 'the datasets could not be read: '.$unread : 'no pool has a dataset it copies'), $unread !== '' ? true : null); + $not = zfsmScheduleCopiesUnderNot($under, $listed); + return $not !== '' ? array('Scheduled copies not made: '.$not, true) : null; +} + +// Where a copy is due here: each this run makes, and each a copy's own schedule makes, which a holder there would refuse. +function zfsmScheduleCopiesComing($job, $jobs, $datasets) { + $coming = array_fill_keys(array_map(function($zdataset) use ($job) { return zfsmScheduleCopiesDest($job, $zdataset); }, $datasets), true); + foreach ($jobs as $other) { + if (!zfsmScheduleCovers($other) && $other['server'] === '') $coming[$other['dest']] = true; + } + return $coming; +} + +// One dataset's copy (zfsmRunReplicate): array(its code, array(its text, type)); an error ends only this one. $wait: what the run has +// left to wait, less what this one waits. +function zfsmScheduleCopiesOne($job, $zdataset, $zdest, $name, &$wait) { + $said = array('', 'normal'); + $tell = function($title, $text, $type) use (&$said) { $said = array(rtrim($text, '.'), $type); }; + $waited = function($seconds) use (&$wait) { $wait = max(0, $wait - $seconds); }; + try { + $code = zfsmRunReplicate($zdataset, $name, $zdest, null, true, '', array('notify' => $tell, 'wait' => $wait, 'waited' => $waited, 'tag' => zfsmKeepTag('', $job['dest'])), $job['keep']); + } catch (Throwable $error) { + zfsmLog('Replication of '.$zdataset.' to '.$zdest.' failed: '.zfsmOneLine($error->getMessage())); + return array(1, array('Replication of '.$zdataset.' to '.$zdest.' failed: '.zfsmOneLine($error->getMessage()), 'warning')); + } + return array($code, $said); +} + // A run's line: how many it copied, those waiting for their parent's copy, the datasets not read, the first problems. function zfsmScheduleCopiesSaid($under, $made, $waiting, $unread, $problems) { return 'Scheduled copies under '.$under.': '.$made.' dataset(s) copied' @@ -120,15 +152,14 @@ function zfsmScheduleCopiesSaid($under, $made, $waiting, $unread, $problems) { .($unread !== '' ? '. Not every dataset could be read: '.$unread : '').($problems ? '. '.zfsmKeepFirst($problems) : ''); } -// Whether the datasets above $zdest are there for its copy: each missing one below $job's dest is made a holder -// (zfsmReplicaHolderMake), but where a dataset this run covers gets its copy. '' when they are, null to wait for that copy, else -// why one could not be made. $here gains each holder made. -function zfsmScheduleCopiesPlace($job, $zdest, $covered, &$here) { +// The datasets above $zdest, below $job's dest: '' when each is there or made a holder, null to wait where a copy is $coming, +// else why a holder could not be made. $here gains each holder. +function zfsmScheduleCopiesPlace($job, $zdest, $coming, &$here) { $level = substr($job['dest'], 0, -1); foreach (array_slice(explode('/', substr($zdest, strlen($job['dest']))), 0, -1) as $part) { $level .= '/'.$part; if (isset($here[$level])) continue; - if (isset($covered[substr($level, strlen($job['dest']))])) return null; + if (isset($coming[$level])) return null; $why = zfsmReplicaHolderMake($level); if ($why !== '') return $why; $here[$level] = true; diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index 8a266801..d894feb4 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -15,7 +15,12 @@ function zfsmScheduleSnapshotValid($job) { return $dataset === '' ? zfsmScheduleRepeats($every) : zfsmValidDatasetName($dataset) && (zfsmScheduleRepeats($every) || $every === 'none'); } -// The test of $job's kind: each dataset follows one snapshot schedule and one of copies under a dataset. +// Whether $job covers datasets below its own: a snapshot schedule or copies under a dataset, not one copy's. +function zfsmScheduleCovers($job) { + return zfsmScheduleSnapshotsOnly($job) || zfsmScheduleCopiesUnder($job); +} + +// The test of the kind of $job, one that covers (zfsmScheduleCovers): each dataset follows one of each kind. function zfsmScheduleKindOf($job) { return zfsmScheduleCopiesUnder($job) ? 'zfsmScheduleCopiesUnder' : 'zfsmScheduleSnapshotsOnly'; } @@ -79,19 +84,19 @@ function zfsmScheduleSnapshotPreview($zdataset, $cfg, $wait = ZFSM_SCHEDULE_COUN define('ZFSM_SCHEDULE_COUNT_WAIT', 10); define('ZFSM_SCHEDULE_COVER_WAIT', 30); -// What a schedule covers, as one answer for zfsmScheduleListed: the datasets, their snapshots' names (both read with $more) and $cfg's -// Snapshots Prefix. $read runs as zfsmReadWithin does. +// What a schedule covers, as one answer for zfsmScheduleListed: the datasets, their snapshots' names (both read with $more), the names +// read's exit code, and $cfg's Snapshots Prefix and a new snapshot's name. $read runs as zfsmReadWithin does. function zfsmScheduleCoverRead($cfg, $wait, $more = array(), $read = 'zfsmReadWithin') { - $listed = $read($wait, 'zfs', array_merge(ZFSM_SCHEDULE_COVERED, $more)) + array('prefix' => (string)($cfg['snap_prefix'] ?? '')); + $listed = $read($wait, 'zfs', array_merge(ZFSM_SCHEDULE_COVERED, $more)) + array('prefix' => (string)($cfg['snap_prefix'] ?? ''), 'name' => zfsmNewSnapshotName($cfg, time())); // None listed (a hung pool waits its whole limit): no dataset for the names to mark, so no second wait. if (!zfsmScheduleListed($listed)[0]) return $listed; - // Names it can't read mark none: those datasets are still taken, and zfsmScheduleCovered says why. + // Names it can't read mark none: a snapshot schedule still takes those datasets, copies leave them out (zfsmScheduleCopiesNot). $names = $read($wait, 'zfs', array_merge(ZFSM_SCHEDULE_SNAPSHOTS, $more)); - return array('code' => $listed['code'] !== 0 ? $listed['code'] : $names['code'], 'output' => $listed['output']."\n".$names['output']) + $listed; + return array('code' => $listed['code'] !== 0 ? $listed['code'] : $names['code'], 'output' => $listed['output']."\n".$names['output'], 'names' => $names['code']) + $listed; } -// $listed (zfsmScheduleCoverRead) as array(readonly, mountpoint, has Sanoid's or Syncoid's snapshots) by dataset, and the lines that -// don't parse. With a Snapshots Prefix that starts as theirs do, a name that starts with it is the plugin's own. +// $listed (zfsmScheduleCoverRead) as array(readonly, mountpoint, has Sanoid's or Syncoid's snapshots: null when its names weren't read) +// by dataset, and the lines that don't parse. With a Snapshots Prefix that starts as theirs do, a name that starts with it is ours. function zfsmScheduleListed($listed) { $rows = $errors = $automatic = array(); $own = zfsmIsAutomaticSnapshot('@'.($listed['prefix'] ?? '')) ? '@'.$listed['prefix'] : ''; @@ -102,7 +107,8 @@ function zfsmScheduleListed($listed) { elseif (count($parts) === 1 && zfsmValidDatasetName($of)) $automatic[$of] = ($automatic[$of] ?? false) || (zfsmIsAutomaticSnapshot($line) && ($own === '' || strpos($line, $of.$own) !== 0)); else $errors[] = $line; } - foreach ($rows as $zdataset => $row) $rows[$zdataset][] = $automatic[$zdataset] ?? false; + $unknown = ($listed['names'] ?? 0) === 0 ? false : null; + foreach ($rows as $zdataset => $row) $rows[$zdataset][] = $automatic[$zdataset] ?? $unknown; // Both reads fail a dataset they can't open: say so once. return array($rows, array_values(array_unique($errors))); } @@ -117,9 +123,9 @@ function zfsmScheduleSkipped($zdataset, $row) { // Why no run of $job takes $zdataset, listed as $row: zfsmScheduleSkipped's reasons, and for copies under a dataset // zfsmScheduleCopiesNot's; '' when one can. -function zfsmScheduleLeftOut($job, $jobs, $zdataset, $row) { +function zfsmScheduleLeftOut($job, $jobs, $zdataset, $row, $name) { $why = zfsmScheduleSkipped($zdataset, $row); - return $why === '' && zfsmScheduleCopiesUnder($job) ? zfsmScheduleCopiesNot($job, $jobs, $zdataset) : $why; + return $why === '' && zfsmScheduleCopiesUnder($job) ? zfsmScheduleCopiesNot($job, $jobs, $zdataset, $row, $name) : $why; } // From $listed: by pool, each dataset that follows $job (zfsmScheduleFollowed of its kind in $jobs) and a run can take @@ -128,7 +134,7 @@ function zfsmScheduleCovered($listed, $job, $jobs) { list($rows, $errors) = zfsmScheduleListed($listed); $pools = array(); foreach ($rows as $zdataset => $row) { - if (zfsmScheduleLeftOut($job, $jobs, $zdataset, $row) === '' && (zfsmScheduleFollowed($zdataset, $jobs, zfsmScheduleKindOf($job))['id'] ?? null) === $job['id']) $pools[explode('/', $zdataset)[0]][] = $zdataset; + if (zfsmScheduleLeftOut($job, $jobs, $zdataset, $row, $listed['name'] ?? '') === '' && (zfsmScheduleFollowed($zdataset, $jobs, zfsmScheduleKindOf($job))['id'] ?? null) === $job['id']) $pools[explode('/', $zdataset)[0]][] = $zdataset; } // zfs lists what it can read, and fails for the rest: a dataset's own minds errors naming none, its datasets or one above. if (($listed['code'] ?? 1) === 0) return array($pools, ''); From d5766a4441ba1a42420fc33dc2d67bc34940de9d Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:02:27 +0800 Subject: [PATCH 3/8] fix(schedules): Replicate Every Dataset skips a start quietly for a day and makes a gone copy again - A start while the run before still goes is logged only, and counts as a problem once that run has gone on a day - A run marks its start on its lock file, which the next start reads - A copy that is gone, as after the pool it goes into is made again, is sent in full again --- tests/schedule_copies_test.php | 25 +++++++++++++++++---- zfs.master/ZFSMasterSettings.page | 4 ++-- zfs.master/include/ZFSMReplicate.php | 5 +++-- zfs.master/include/ZFSMScheduleCopies.php | 7 ++++-- zfs.master/include/ZFSMScheduleSnapshot.php | 22 ++++++++++++++---- 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index e53781e5..6a9fccfe 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -164,6 +164,13 @@ function same($expected, $actual, $message) { same(6, count(preg_grep('/^send -P -v -L -i /', $read('calls.log'))), 'each sends its changes'); same(array(), $read('notify.log'), 'only logged'); +// A copy gone, as when the pool it goes into was made again: the next run makes it again in full. +$pool('datasets', array_values(array_filter($read('pool/datasets'), function($line) { return strpos($line, "backup/usb\t") !== 0; }))); +$pool('marks', array_values(array_filter($read('pool/marks'), function($line) { return strpos($line, 'backup/usb@') !== 0; }))); +$reset(); +same(0, zfsmScheduleRun($id, $cfg, $t + 9000), 'a run with a copy gone'); +same(array(true, 1), array(in_array('backup/usb', $datasets(), true), count(preg_grep('/^send -P -v -L usb@/', $read('calls.log')))), 'makes it again in full'); + // tank/san no longer Sanoid's: its copy's place holds a holder, which it can't replace; tank/san/kid goes on. $pool('marks', array_values(array_filter($read('pool/marks'), function($line) { return strpos($line, 'tank/san@autosnap_') !== 0; }))); $reset(); @@ -186,13 +193,23 @@ function same($expected, $actual, $message) { same(true, time() - $started < 8, 'in all, its budget and the slow send'); same(1, preg_match('/^warning: Scheduled copies under backup: 4 of 6 dataset\(s\) copied\. Replication of tank\/data to backup\/tank\/data not started: another replication of this dataset was still running after this run waited 1 minute\(s\) for it; Replication of tank\/docker to backup\/tank\/docker not started: another replication of this dataset was still running, and the run had no time left to wait for it/', $read('notify-detail.log')[0] ?? ''), 'each says so'); -// A run while the one before still goes: none made, notified. +// A run marks its start on its lock file, which the next start reads. +$file = zfsmJobDir('schedule').'/'.$id.'.lock'; +touch($file, time() - 90000); +$before = time(); +zfsmScheduleRun($id, $cfg, $t + 17400); +clearstatcache(); +same(true, filemtime($file) >= $before, 'a run marks its start'); +// A start while the run before still goes: none made, only logged until that one has gone on a day, then notified. $reset(); @unlink(zfsmScheduleStateFile($id)); -$held = zfsmJobLock(zfsmJobDir('schedule').'/'.$id.'.lock'); -same(1, zfsmScheduleRun($id, $cfg, $t + 18000), 'a run while the one before goes'); +$held = zfsmJobLock($file); +same(0, zfsmScheduleRun($id, $cfg, $t + 18000), 'a start while the run before goes'); +same(array(array(), 1, 0), array($read('notify.log'), $logged('/ Schedule '.$id.': Scheduled copies not made: the run before is still going$/'), zfsmScheduleFailures($id)), 'only logged'); +touch($file, time() - 90000); +same(1, zfsmScheduleRun($id, $cfg, $t + 18060), 'a start a day after it'); zfsmJobUnlock($held); -same(array(array('warning: ZFS Replication'), 1), array($read('notify.log'), preg_match('/^warning: Scheduled copies not made: the run before is still going/', $read('notify-detail.log')[0] ?? '')), 'notified'); +same(array(array('warning: ZFS Replication'), 1), array($read('notify.log'), preg_match('/^warning: Scheduled copies not made: the run before is still going, 25 hour\(s\) after it started/', $read('notify-detail.log')[0] ?? '')), 'notified'); // An error in one dataset's replication ends only that one: the datasets below it wait, usb is copied. $fresh(); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index c93cc480..e3edcc56 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -345,12 +345,12 @@ _(Replicate Every Dataset)_: :zc_settings_copysched_plug: > Copies every dataset in every pool into the dataset you name under **Copy Under**, on this server, each copy named for its dataset: with `backup`, `cache/appdata` is copied to `backup/cache/appdata`. It runs every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. Save changes it, and the answer says how many datasets it copies. Off stops it; the copies and the snapshots it made stay. > -> - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full. The copies are read-only and not mounted. +> - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full, as does a run after a copy is gone, such as when the pool it goes into was made again. The copies are read-only and not mounted. > - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy, one whose name is too long for the bookmark its copy keeps (over about 70 characters), and, when a run can't read the snapshot names, those it couldn't read: they may be left to Sanoid or Syncoid. > - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. When the copy of the dataset above one is missing because it failed, or its own copy schedule has not made it yet, that one waits for the next run. > - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. -> - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. So does a run that falls due while the one before still goes, as a first run can. +> - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. A run that falls due while the one before still goes, as a first run can, is skipped and logged; once the one before has gone on for a day, it counts as a problem. > - Each run sends one notification at most: what it copied and its first problems. The first run with a problem sends it; later ones are logged in `/var/log/zfs.master.log` until a run works again, and that run's notification says so. The log has each dataset's replication. :end diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index cf2e81fd..5071bdea 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -138,7 +138,8 @@ function zfsmReplicaMark($zdataset, $plan) { // $take: the run takes $snapshot first, under its lock, and removes it again unless a copy holds it or a stopped run may need it. // $started runs once the plan holds, before zfs send; until then the reason a run doesn't start also goes to the starting page. // $schedule, for a scheduled run (zfsmScheduleRun): its 'notify'(title, text, importance), the seconds to 'wait' out another run, and no full send to a copy that is gone; -// optional: 'waited'(seconds) told how long it waited, what to do when the dataset is 'gone', and a 'tag' (zfsmKeepTag) for its holds and Keep. +// optional: 'waited'(seconds) told how long it waited, what to do when the dataset is 'gone', a 'tag' (zfsmKeepTag) for its holds and Keep, +// and 'remake' to send a copy that is gone in full again. // $keep, for any run of a schedule: the snapshot it takes is tagged for the copy (zfsmKeepTag), and once it lands only those of // them $keep keeps stay (0: all, zfsmKeepPrune). function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = false, $server = '', $schedule = null, $keep = null) { @@ -177,7 +178,7 @@ function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = $target = zfsmReplicaTargetOpen($server); $target['zstd'] = zfsmReplicaTargetZstd($target); for ($pass = 1; ; $pass++) { - $plan = zfsmReplicatePlan($zdataset, $snapshot, $zdest, array('estimate' => false, 'target' => $target, 'scheduled' => $schedule !== null)); + $plan = zfsmReplicatePlan($zdataset, $snapshot, $zdest, array('estimate' => false, 'target' => $target, 'scheduled' => $schedule !== null && !($schedule['remake'] ?? false))); if (!$plan['can_proceed']) return $refused($plan['error']); if ($started !== null) $started(); list($code, $text, $type) = zfsmReplicateSent($plan, $zdataset, $snapshot, $take, $target, $tag); diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index 738038c7..e173dfa3 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -2,6 +2,9 @@ // Copies under a dataset: records whose dest, a dataset here and '/', holds each covered dataset's copy (backup/ and cache/appdata: // backup/cache/appdata). Dataset '' is Settings' Replicate Every Dataset; None is dest '/'. Coverage: zfsmScheduleFollowed. +// A start while the run before still goes is only logged for a day (zfsmScheduleLockedSkip): a first run sends each dataset in full. +define('ZFSM_SCHEDULE_COPIES_GRACE', 86400); + function zfsmScheduleCopiesUnder($job) { return is_string($job['dest'] ?? null) && substr($job['dest'], -1) === '/'; } @@ -76,7 +79,7 @@ function zfsmScheduleCopiesSaved($job, $listed, $after) { function zfsmScheduleCopiesRun($job, $jobs, $cfg, $now, $notify, $wait = ZFSM_SCHEDULE_WAIT) { return zfsmScheduleLockedRun($job, 'ZFS Replication', 'Scheduled copies not made: ', $notify, function() use ($job, $jobs, $cfg, $now, $wait) { return zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait); - }); + }, ZFSM_SCHEDULE_COPIES_GRACE); } // What a run copied, and whether it had a problem: null when zfs read every pool and none had a dataset to copy. Its waits for other @@ -136,7 +139,7 @@ function zfsmScheduleCopiesOne($job, $zdataset, $zdest, $name, &$wait) { $tell = function($title, $text, $type) use (&$said) { $said = array(rtrim($text, '.'), $type); }; $waited = function($seconds) use (&$wait) { $wait = max(0, $wait - $seconds); }; try { - $code = zfsmRunReplicate($zdataset, $name, $zdest, null, true, '', array('notify' => $tell, 'wait' => $wait, 'waited' => $waited, 'tag' => zfsmKeepTag('', $job['dest'])), $job['keep']); + $code = zfsmRunReplicate($zdataset, $name, $zdest, null, true, '', array('notify' => $tell, 'wait' => $wait, 'waited' => $waited, 'tag' => zfsmKeepTag('', $job['dest']), 'remake' => true), $job['keep']); } catch (Throwable $error) { zfsmLog('Replication of '.$zdataset.' to '.$zdest.' failed: '.zfsmOneLine($error->getMessage())); return array(1, array('Replication of '.$zdataset.' to '.$zdest.' failed: '.zfsmOneLine($error->getMessage()), 'warning')); diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index d894feb4..885daaf1 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -291,10 +291,14 @@ function zfsmScheduleSnapshotRun($job, $jobs, $cfg, $now, $notify, $wait = ZFSM_ } // $run of $job (array(text, problem) as zfsmScheduleSnapshotTell takes them) under the job's lock; $not starts what a run that can't -// start says. One at a time: a run still going at the next one's time (a long Keep, a hung pool) stops that one, so runs don't pile up. -function zfsmScheduleLockedRun($job, $title, $not, $notify, $run) { - $lock = zfsmJobLock(zfsmJobDir('schedule').'/'.$job['id'].'.lock'); - if (!is_resource($lock)) return zfsmScheduleSnapshotTell($job, $not.($lock === null ? 'the lock file could not be opened' : 'the run before is still going'), true, $notify, $title); +// start says. One at a time: a run still going at the next one's time (a long send, a hung pool) stops that one (zfsmScheduleLockedSkip). +function zfsmScheduleLockedRun($job, $title, $not, $notify, $run, $grace = 0) { + $file = zfsmJobDir('schedule').'/'.$job['id'].'.lock'; + $lock = zfsmJobLock($file); + if ($lock === null) return zfsmScheduleSnapshotTell($job, $not.'the lock file could not be opened', true, $notify, $title); + if ($lock === false) return zfsmScheduleLockedSkip($job, $title, $not, $notify, $file, $grace); + // Its time is this run's start, which the next start reads. + @touch($file); try { list($text, $problem) = $run(); } finally { @@ -303,6 +307,16 @@ function zfsmScheduleLockedRun($job, $title, $not, $notify, $run) { return zfsmScheduleSnapshotTell($job, $text, $problem, $notify, $title); } +// A start while the run before still goes: only logged until that run, which started at $file's time, has gone on $grace seconds. +function zfsmScheduleLockedSkip($job, $title, $not, $notify, $file, $grace) { + $age = time() - (int)@filemtime($file); + if ($age < $grace) { + zfsmLog('Schedule '.$job['id'].': '.$not.'the run before is still going', LOG_NOTICE); + return 0; + } + return zfsmScheduleSnapshotTell($job, $not.'the run before is still going'.($grace > 0 ? ', '.intdiv($age, 3600).' hour(s) after it started' : ''), true, $notify, $title); +} + // What a run took and kept, and whether it had a problem: null when zfs read every pool and the default had no dataset to take. function zfsmScheduleSnapshotsTaken($job, $jobs, $cfg, $now, $wait) { $listed = zfsmScheduleCoverRead($cfg, $wait); From 5e9cca702fd66434071a427d065e2316ebc7ac64 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:12:47 +0800 Subject: [PATCH 4/8] fix(schedules): say when the snapshot names, not the datasets, could not be read - A run that copies none because the names read failed says so - The help gives the bookmark limit for backup, and a shorter one under a longer Copy Under - A test reads what a schedule covers as a run does, with a new snapshot name --- tests/schedule_copies_test.php | 2 +- tests/schedule_snapshot_test.php | 2 +- zfs.master/ZFSMasterSettings.page | 2 +- zfs.master/include/ZFSMScheduleCopies.php | 3 ++- zfs.master/include/ZFSMScheduleSnapshot.php | 9 +++++++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 6a9fccfe..ef8787fd 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -232,7 +232,7 @@ function same($expected, $actual, $message) { same(1, zfsmScheduleRun($id, $cfg, $t + 20400), 'a run without the names'); putenv('ZFSM_TEST_NAMES_FAIL'); same(array(array('backup', 'backup/old'), array()), array(array_values(preg_grep('~^backup~', $datasets())), preg_grep('/^(snapshot|send|create) /', $read('calls.log'))), 'nothing taken or copied'); -same(1, preg_match('/^warning: Scheduled copies not made: the datasets could not be read: cannot iterate filesystems: Input\/output error/', $read('notify-detail.log')[0] ?? ''), 'saying why'); +same(1, preg_match('/^warning: Scheduled copies not made: the snapshot names could not be read: cannot iterate filesystems: Input\/output error/', $read('notify-detail.log')[0] ?? ''), 'saying why'); // Copy Under gone: none made; a later problem, so only logged. $pool('datasets', array_values(array_filter($read('pool/datasets'), function($line) { return strpos($line, 'backup') !== 0; }))); diff --git a/tests/schedule_snapshot_test.php b/tests/schedule_snapshot_test.php index a3d07ff2..9f0975ba 100644 --- a/tests/schedule_snapshot_test.php +++ b/tests/schedule_snapshot_test.php @@ -58,7 +58,7 @@ function same($expected, $actual, $message) { $records = array($every, array('dataset' => 'tank/data') + $every, array('dataset' => 'tank/data', 'every' => 'none') + $every, array('server' => 'nas') + $every, array('every' => 'none') + $every, array('dataset' => 'tank/bad name!') + $every, array('every' => array('hourly')) + $every, $copy, array('every' => 'none') + $copy); same(array(true, true, true, false, false, false, false, true, false), array_map('zfsmScheduleValid', $records), 'which records are snapshot schedules'); -same(array(array('tank' => array('tank', 'tank/data', 'tank/docker', 'tank/vm'), 'backup' => array('backup')), ''), zfsmScheduleCovered(runProcess('zfs', ZFSM_SCHEDULE_COVERED), $every, array($every)), 'it covers every dataset but read-only and legacy ones, by pool'); +same(array(array('tank' => array('tank', 'tank/data', 'tank/docker', 'tank/vm'), 'backup' => array('backup')), ''), zfsmScheduleCovered(zfsmScheduleCoverRead(array(), 30), $every, array($every)), 'it covers every dataset but read-only and legacy ones, by pool'); // Off with none saves none; Settings shows Off. same(array('saved' => true, 'why' => '', 'said' => 'Snapshot Every Dataset is off; the snapshots it took stay'), zfsmScheduleSnapshotSave('', 'off', '', '', 0, $cfg), 'off with none'); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index e3edcc56..42289789 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -346,7 +346,7 @@ _(Replicate Every Dataset)_: > Copies every dataset in every pool into the dataset you name under **Copy Under**, on this server, each copy named for its dataset: with `backup`, `cache/appdata` is copied to `backup/cache/appdata`. It runs every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. Save changes it, and the answer says how many datasets it copies. Off stops it; the copies and the snapshots it made stay. > > - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full, as does a run after a copy is gone, such as when the pool it goes into was made again. The copies are read-only and not mounted. -> - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy, one whose name is too long for the bookmark its copy keeps (over about 70 characters), and, when a run can't read the snapshot names, those it couldn't read: they may be left to Sanoid or Syncoid. +> - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy, one whose name is too long for the bookmark its copy keeps (over about 70 characters under `backup`, fewer under a longer **Copy Under**), and, when a run can't read the snapshot names, those it couldn't read: they may be left to Sanoid or Syncoid. > - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. When the copy of the dataset above one is missing because it failed, or its own copy schedule has not made it yet, that one waits for the next run. > - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index e173dfa3..b07e751c 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -118,7 +118,8 @@ function zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait) { // What a run says when it copies none, as zfsmScheduleCopiesMade answers; null when it can copy. function zfsmScheduleCopiesNone($under, $listed, $pools, $unread) { - if (!$pools) return array('Scheduled copies not made: '.($unread !== '' ? 'the datasets could not be read: '.$unread : 'no pool has a dataset it copies'), $unread !== '' ? true : null); + $what = ($listed['names'] ?? 0) !== 0 ? 'the snapshot names' : 'the datasets'; + if (!$pools) return array('Scheduled copies not made: '.($unread !== '' ? $what.' could not be read: '.$unread : 'no pool has a dataset it copies'), $unread !== '' ? true : null); $not = zfsmScheduleCopiesUnderNot($under, $listed); return $not !== '' ? array('Scheduled copies not made: '.$not, true) : null; } diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index 885daaf1..a56b8e4d 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -104,7 +104,7 @@ function zfsmScheduleListed($listed) { $parts = explode("\t", $line, 3); $of = (string)strstr($line, '@', true); if (count($parts) === 3 && zfsmValidDatasetName($parts[0])) $rows[$parts[0]] = array($parts[1], $parts[2]); - elseif (count($parts) === 1 && zfsmValidDatasetName($of)) $automatic[$of] = ($automatic[$of] ?? false) || (zfsmIsAutomaticSnapshot($line) && ($own === '' || strpos($line, $of.$own) !== 0)); + elseif (count($parts) === 1 && zfsmValidDatasetName($of)) $automatic[$of] = ($automatic[$of] ?? false) || zfsmScheduleTheirs($line, $of, $own); else $errors[] = $line; } $unknown = ($listed['names'] ?? 0) === 0 ? false : null; @@ -113,6 +113,11 @@ function zfsmScheduleListed($listed) { return array($rows, array_values(array_unique($errors))); } +// Whether snapshot $line of $of is Sanoid's or Syncoid's: not ours, under a Snapshots Prefix that starts as theirs do ($own). +function zfsmScheduleTheirs($line, $of, $own) { + return zfsmIsAutomaticSnapshot($line) && ($own === '' || strpos($line, $of.$own) !== 0); +} + // Why no run takes $zdataset, listed as $row (zfsmScheduleListed); '' when one can. function zfsmScheduleSkipped($zdataset, $row) { if ($row[0] === 'on') return $zdataset.' is read-only, as a copy received here is: a snapshot in it stops the next replication into it'; @@ -134,7 +139,7 @@ function zfsmScheduleCovered($listed, $job, $jobs) { list($rows, $errors) = zfsmScheduleListed($listed); $pools = array(); foreach ($rows as $zdataset => $row) { - if (zfsmScheduleLeftOut($job, $jobs, $zdataset, $row, $listed['name'] ?? '') === '' && (zfsmScheduleFollowed($zdataset, $jobs, zfsmScheduleKindOf($job))['id'] ?? null) === $job['id']) $pools[explode('/', $zdataset)[0]][] = $zdataset; + if (zfsmScheduleLeftOut($job, $jobs, $zdataset, $row, $listed['name']) === '' && (zfsmScheduleFollowed($zdataset, $jobs, zfsmScheduleKindOf($job))['id'] ?? null) === $job['id']) $pools[explode('/', $zdataset)[0]][] = $zdataset; } // zfs lists what it can read, and fails for the rest: a dataset's own minds errors naming none, its datasets or one above. if (($listed['code'] ?? 1) === 0) return array($pools, ''); From c96a2def08b075a76bf30c7dbf24469f7745318c Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:13:40 +0800 Subject: [PATCH 5/8] style(replication): the run's schedule options in one comment line --- zfs.master/include/ZFSMReplicate.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index 5071bdea..c67dc18e 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -138,8 +138,7 @@ function zfsmReplicaMark($zdataset, $plan) { // $take: the run takes $snapshot first, under its lock, and removes it again unless a copy holds it or a stopped run may need it. // $started runs once the plan holds, before zfs send; until then the reason a run doesn't start also goes to the starting page. // $schedule, for a scheduled run (zfsmScheduleRun): its 'notify'(title, text, importance), the seconds to 'wait' out another run, and no full send to a copy that is gone; -// optional: 'waited'(seconds) told how long it waited, what to do when the dataset is 'gone', a 'tag' (zfsmKeepTag) for its holds and Keep, -// and 'remake' to send a copy that is gone in full again. +// optional: 'waited'(seconds) told the wait, 'gone' what to do when the dataset is, a 'tag' (zfsmKeepTag) for holds and Keep, 'remake' a gone copy in full. // $keep, for any run of a schedule: the snapshot it takes is tagged for the copy (zfsmKeepTag), and once it lands only those of // them $keep keeps stay (0: all, zfsmKeepPrune). function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = false, $server = '', $schedule = null, $keep = null) { From beb6650e3ea97594a3dedcba296693e1db147f03 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:32:11 +0800 Subject: [PATCH 6/8] fix(schedules): a snapshot-names read that timed out stays unknown, so nothing is copied --- tests/schedule_copies_test.php | 4 ++++ zfs.master/include/ZFSMScheduleCopies.php | 2 +- zfs.master/include/ZFSMScheduleSnapshot.php | 8 +++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index ef8787fd..44dcb892 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -233,6 +233,10 @@ function same($expected, $actual, $message) { putenv('ZFSM_TEST_NAMES_FAIL'); same(array(array('backup', 'backup/old'), array()), array(array_values(preg_grep('~^backup~', $datasets())), preg_grep('/^(snapshot|send|create) /', $read('calls.log'))), 'nothing taken or copied'); same(1, preg_match('/^warning: Scheduled copies not made: the snapshot names could not be read: cannot iterate filesystems: Input\/output error/', $read('notify-detail.log')[0] ?? ''), 'saying why'); +// One that timed out (zfsmReadWithin's null code) is not read either. +$timed = zfsmScheduleCoverRead($cfg, 5, array(), function($wait, $command, $args) { return in_array('snapshot', $args, true) ? array('code' => null, 'output' => 'zfs did not answer in 5 seconds') : zfsmReadWithin($wait, $command, $args); }); +same(array(array(), null), array(zfsmScheduleCovered($timed, $every, array($every))[0], zfsmScheduleListed($timed)[0]['tank'][2]), 'none copied after a names read that timed out'); +same(array('Scheduled copies not made: the snapshot names could not be read: zfs did not answer in 5 seconds', true), zfsmScheduleCopiesNone('backup', $timed, array(), zfsmScheduleCovered($timed, $every, array($every))[1]), 'saying why, not that the datasets were unread'); // Copy Under gone: none made; a later problem, so only logged. $pool('datasets', array_values(array_filter($read('pool/datasets'), function($line) { return strpos($line, 'backup') !== 0; }))); diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index b07e751c..f9c00813 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -118,7 +118,7 @@ function zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait) { // What a run says when it copies none, as zfsmScheduleCopiesMade answers; null when it can copy. function zfsmScheduleCopiesNone($under, $listed, $pools, $unread) { - $what = ($listed['names'] ?? 0) !== 0 ? 'the snapshot names' : 'the datasets'; + $what = zfsmScheduleNamesUnread($listed) ? 'the snapshot names' : 'the datasets'; if (!$pools) return array('Scheduled copies not made: '.($unread !== '' ? $what.' could not be read: '.$unread : 'no pool has a dataset it copies'), $unread !== '' ? true : null); $not = zfsmScheduleCopiesUnderNot($under, $listed); return $not !== '' ? array('Scheduled copies not made: '.$not, true) : null; diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index a56b8e4d..81e28dd5 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -107,12 +107,18 @@ function zfsmScheduleListed($listed) { elseif (count($parts) === 1 && zfsmValidDatasetName($of)) $automatic[$of] = ($automatic[$of] ?? false) || zfsmScheduleTheirs($line, $of, $own); else $errors[] = $line; } - $unknown = ($listed['names'] ?? 0) === 0 ? false : null; + $unknown = zfsmScheduleNamesUnread($listed) ? null : false; foreach ($rows as $zdataset => $row) $rows[$zdataset][] = $automatic[$zdataset] ?? $unknown; // Both reads fail a dataset they can't open: say so once. return array($rows, array_values(array_unique($errors))); } +// Whether $listed (zfsmScheduleCoverRead) read datasets but not their snapshots' names: only a read that exited 0 did, and one that +// timed out has a null code. +function zfsmScheduleNamesUnread($listed) { + return array_key_exists('names', $listed) && $listed['names'] !== 0; +} + // Whether snapshot $line of $of is Sanoid's or Syncoid's: not ours, under a Snapshots Prefix that starts as theirs do ($own). function zfsmScheduleTheirs($line, $of, $own) { return zfsmIsAutomaticSnapshot($line) && ($own === '' || strpos($line, $of.$own) !== 0); From e9e8bad344eccbb9657bf42f4c1b41c743ec5a14 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 09:30:03 +0800 Subject: [PATCH 7/8] fix(schedules): a scheduled run holds snapshots by the copy, so carrying it on by hand takes the holds off --- tests/schedule_copies_test.php | 10 ++++++++++ zfs.master/include/ZFSMReplicate.php | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 44dcb892..75a53c27 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -180,6 +180,16 @@ function same($expected, $actual, $message) { same(array(), $snaps('tank/san'), 'the snapshot it took for it goes'); same(array("backup/tank/san/kid@$o"), array_keys($snaps('backup/tank/san/kid')), 'the one below is copied'); +// One cut off part-way holds for each copy, as a run by hand does: carried on by hand, its holds come off. +putenv('ZFSM_TEST_SEND_CUT=1'); +zfsmScheduleRun($id, $cfg, $t + 12600); +putenv('ZFSM_TEST_SEND_CUT'); +$p = $name($t + 12600); +$hold = zfsmKeepHoldTag(zfsmKeepTag('', 'backup/tank/data')); +same(array("tank/data@$p\t$hold", "tank/data@$o\t$hold"), array_values(preg_grep('~^tank/data@~', $read('pool/holds'))), 'a run cut off holds for the copy'); +same(0, zfsmRunReplicate('tank/data', $p, 'backup/tank/data'), 'carried on by hand'); +same(array(), preg_grep('~^tank/data@~', $read('pool/holds')), 'which takes its holds off'); + // Other replications of tank/data and tank/docker still running, after tank's send outlasted the budget: only waits use it, and // they share it, so the second has none left. $fresh(); diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index c67dc18e..589b7830 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -138,7 +138,7 @@ function zfsmReplicaMark($zdataset, $plan) { // $take: the run takes $snapshot first, under its lock, and removes it again unless a copy holds it or a stopped run may need it. // $started runs once the plan holds, before zfs send; until then the reason a run doesn't start also goes to the starting page. // $schedule, for a scheduled run (zfsmScheduleRun): its 'notify'(title, text, importance), the seconds to 'wait' out another run, and no full send to a copy that is gone; -// optional: 'waited'(seconds) told the wait, 'gone' what to do when the dataset is, a 'tag' (zfsmKeepTag) for holds and Keep, 'remake' a gone copy in full. +// optional: 'waited'(seconds) told the wait, 'gone' what to do when the dataset is, a 'tag' (zfsmKeepTag) for Keep, 'remake' a gone copy in full. // $keep, for any run of a schedule: the snapshot it takes is tagged for the copy (zfsmKeepTag), and once it lands only those of // them $keep keeps stay (0: all, zfsmKeepPrune). function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = false, $server = '', $schedule = null, $keep = null) { @@ -180,7 +180,8 @@ function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = $plan = zfsmReplicatePlan($zdataset, $snapshot, $zdest, array('estimate' => false, 'target' => $target, 'scheduled' => $schedule !== null && !($schedule['remake'] ?? false))); if (!$plan['can_proceed']) return $refused($plan['error']); if ($started !== null) $started(); - list($code, $text, $type) = zfsmReplicateSent($plan, $zdataset, $snapshot, $take, $target, $tag); + // Holds go by the copy, so a run by hand carries on a scheduled one and takes its holds off. + list($code, $text, $type) = zfsmReplicateSent($plan, $zdataset, $snapshot, $take, $target, zfsmKeepTag($server, $zdest)); // A scheduled run that carried on a stopped one then sends the snapshot it took (help: Schedules); only its last step notifies. if ($code === 0 && $schedule !== null && $plan['mode'] === 'resume' && $pass === 1) continue; list($text, $type) = zfsmKeepAfter($code, $zdataset, $plan, $keep, $tag, $to, $text, $type); From dc5fbcd40f31b888f12d5838052657bb9d70dd4a Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:37:08 +0800 Subject: [PATCH 8/8] style(schedules): the Settings list comment back over zfsmSchedulesHtml --- zfs.master/include/ZFSMSchedule.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zfs.master/include/ZFSMSchedule.php b/zfs.master/include/ZFSMSchedule.php index 38ab52e4..b5ad95c7 100644 --- a/zfs.master/include/ZFSMSchedule.php +++ b/zfs.master/include/ZFSMSchedule.php @@ -376,7 +376,6 @@ function zfsmScheduleRun($id, $cfg, $now, $wait = ZFSM_SCHEDULE_WAIT) { } } -// Settings' list; $cfg for Snapshots Pattern, which can change after a schedule was saved. // Settings' Days boxes of the row whose ids start $prefix, $days ticked. function zfsmScheduleDaysHtml($prefix, $days) { $html = ''; @@ -386,6 +385,7 @@ function zfsmScheduleDaysHtml($prefix, $days) { return ''.$html.''; } +// Settings' list; $cfg for Snapshots Pattern, which can change after a schedule was saved. function zfsmSchedulesHtml($cfg) { $read = zfsmScheduleRead(); if ($read === null) return ''.zfsmHtml(zfsmScheduleUnreadable()).'';