From 2668aca548d74dcb9c4609023f062f42a90c3262 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 08:44:24 +0800 Subject: [PATCH 01/24] feat(replication): a dataset's first copy replaces the empty holder in its place and keeps the copies below it --- .coderabbit.yaml | 2 +- tests/fixtures/bin/zfs | 24 +++++++- tests/frontend_impact_test.cjs | 2 + tests/replicate_test.php | 2 +- tests/schedule_copies_test.php | 49 +++++++++++++-- tests/schedule_snapshot_test.php | 2 +- zfs.master/ZFSMasterSettings.page | 2 +- zfs.master/frontend/ZFSMImpact.js | 1 + zfs.master/include/ZFSMReplicaPlan.php | 77 ++++++++++++++++-------- zfs.master/include/ZFSMReplicaTarget.php | 6 ++ zfs.master/include/ZFSMReplicate.php | 4 +- 11 files changed, 132 insertions(+), 39 deletions(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index ebc78cab..97a16232 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -38,7 +38,7 @@ reviews: - "Inside a dataset" means equal to it, or starting with it plus `/` (`@` or `#` for its snapshots and bookmarks). Flag a bare string-prefix test: `tank/data2` is not inside `tank/data`. - Escape anything echoed into HTML or inline script. Config is saved to `/boot`, so saved values from older versions must keep working. - File names, ZFS user properties and anything from another server may not be valid UTF-8. htmlspecialchars() needs ENT_SUBSTITUTE, or it returns '' and the text disappears. json_encode() needs JSON_INVALID_UTF8_SUBSTITUTE, or its false result must be handled; never echo, publish or hash a json_encode() that can fail. - - Every `zfs receive` takes `-s -u -o readonly=on` and never `-F`. A copy of a volume must not appear as a block device (volmode=none). + - Every `zfs receive` takes `-s -u -o readonly=on`, and `-F` only onto a holder that holds no snapshot (zfsmReplicaPlanHolder, zfsmReplicaPlanResume), which zfs needs to replace it. A copy of a volume must not appear as a block device (volmode=none). - Replicate Every Dataset (include/ZFSMScheduleCopies.php) makes an empty holder (`zfs create -o canmount=off -o readonly=on -o zfs.master:holder=on`) only where a copy's parent is missing. The property inherits, so a dataset is a holder only where its source is `local` (zfsmReplicaHolder). - path: "zfs.master/{include/ZFSMDiag*.php,scripts/diagnostics.php}" instructions: | diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index 50caf814..a4fa1c38 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -147,6 +147,7 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ]; then exit 0 ;; receive) dest=$last_arg + force=0; contains_arg -F "$@" && force=1 # ZFSM_TEST_RECV_FAIL=early: fails before it reads, as a receive that stops part-way leaves the rest unread. [ "${ZFSM_TEST_RECV_FAIL:-0}" = early ] && { printf 'cannot receive new filesystem stream: out of space\n' >&2; exit 1; } stream=$(tail -c +313) @@ -156,7 +157,10 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ]; then [ "${ZFSM_TEST_RECV_FAIL:-0}" = 1 ] && { printf 'cannot receive new filesystem stream: out of space\n' >&2; exit 1; } snap=$(printf '%s' "$header" | cut -f2); base=$(printf '%s' "$header" | cut -f3); guid=$(printf '%s' "$header" | cut -f4) [ "${ZFSM_TEST_RECV_FAIL:-0}" = changes ] && [ "$base" != - ] && [ "$base" != RESUME ] && { printf 'cannot receive incremental stream: out of space\n' >&2; exit 1; } - if [ "$base" = - ] && exists "$dest" && [ "$(prop "$dest" receive_resume_token -)" = - ]; then printf "cannot receive new filesystem stream: destination '%s' exists\n" "$dest" >&2; exit 1; fi + # A full stream onto one that is there needs -F; one cut off there carries on only with -F too (2.4.4, live). + if [ "$base" = - ] && exists "$dest" && [ "$(prop "$dest" receive_resume_token -)" = - ] && [ "$force" = 0 ]; then printf "cannot receive new filesystem stream: destination '%s' exists\n" "$dest" >&2; exit 1; fi + if [ "$base" = RESUME ] && [ "$(prop "$dest" zfsm_test_onto -)" = 1 ] && [ "$force" = 0 ]; then printf "cannot receive resume stream: Resuming recv on existing destination '%s'\nmust specify -F to overwrite it\n" "$dest" >&2; exit 1; fi + onto=0; [ "$base" = - ] && exists "$dest" && onto=1 # zfs makes a new copy only inside a dataset that exists. exists "$dest" || exists "${dest%/*}" || { printf "cannot receive new filesystem stream: parent of '%s' does not exist\n" "$dest" >&2; exit 1; } if [ "$base" != - ] && [ "$base" != RESUME ] && [ "$(newest "$dest")" != "$(mark "$base" 2)" ]; then @@ -165,12 +169,30 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ]; then exists "$dest" || printf '%s\t%s\n' "$dest" "$(awk -F'\t' -v n="${snap%@*}" '$1 == n { print $2 }' "$R/datasets")" >> "$R/datasets" if ! printf '%s\n' "$stream" | grep -q '^ZFSMEND$'; then set_prop "$dest" receive_resume_token "tok:$snap" + [ "$onto" = 1 ] && set_prop "$dest" zfsm_test_onto 1 printf 'cannot receive incremental stream: checksum mismatch or incomplete stream.\nPartially received snapshot is saved.\n' >&2; exit 1 fi printf '%s@%s\t%s\t%s\n' "$dest" "${snap#*@}" "$guid" "$(next_txg)" >> "$R/marks" # A received snapshot keeps when the sender took it, as the stream says. creation=$(printf '%s' "$header" | cut -f6); [ -z "$creation" ] || set_prop "$dest@${snap#*@}" creation "$creation" set_prop "$dest" receive_resume_token - + set_prop "$dest" zfsm_test_onto - + exit 0 ;; + inherit) + exists "$last_arg" || { printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1; } + [ "${ZFSM_TEST_INHERIT_FAIL:-0}" = 1 ] && { printf "cannot inherit %s for '%s': permission denied\n" "$2" "$last_arg" >&2; exit 1; } + awk -F'\t' -v n="$last_arg" -v p="$2" '!($1 == n && $2 == p)' "$R/props" > "$R/props.new" && mv "$R/props.new" "$R/props" + exit 0 ;; + rename) + # Moves it and all below it, with their snapshots, bookmarks, properties and holds; ZFSM_TEST_RENAME_FAIL names one that is busy. + eval "from=\${$(($# - 1))}" + exists "$from" || { printf "cannot open '%s': dataset does not exist\n" "$from" >&2; exit 1; } + [ "${ZFSM_TEST_RENAME_FAIL:-}" = "$from" ] && { printf "cannot rename '%s': dataset is busy\n" "$from" >&2; exit 1; } + exists "$last_arg" && { printf "cannot rename '%s': dataset already exists\n" "$from" >&2; exit 1; } + exists "${last_arg%/*}" || { printf "cannot create '%s': parent does not exist\n" "$last_arg" >&2; exit 1; } + for f in datasets marks props holds; do + awk -F'\t' -v OFS='\t' -v a="$from" -v b="$last_arg" '{ n = length(a); c = substr($1, n + 1, 1); if (substr($1, 1, n) == a && (c == "" || c == "/" || c == "@" || c == "#")) $1 = b substr($1, n + 1); print }' "$R/$f" > "$R/$f.new" && mv "$R/$f.new" "$R/$f" + done exit 0 ;; bookmark) guid=$(mark "$2" 2); txg=$(mark "$2" 3) diff --git a/tests/frontend_impact_test.cjs b/tests/frontend_impact_test.cjs index ac6f9b6e..e2696745 100644 --- a/tests/frontend_impact_test.cjs +++ b/tests/frontend_impact_test.cjs @@ -94,6 +94,8 @@ assert.match(context.zfsmReplicatePlanHtml({can_proceed: true, mode: 'incrementa assert.equal(context.zfsmReplicatePlanHtml({can_proceed: false, error: ''}), '', 'no destination yet: no plan'); assert.match(context.zfsmReplicatePlanHtml({can_proceed: false, error: 'The copy goes '}), /color:#e22828[\s\S]*The copy goes <inside>/, 'a refusal shows, escaped'); assert.match(context.zfsmReplicatePlanHtml({can_proceed: true, mode: 'full', size: 2048, raw: false}), /The first replication sends everything, about 2 KiB\./); +assert.match(context.zfsmReplicatePlanHtml({can_proceed: true, mode: 'full', size: null, holder: true}), /It replaces the empty dataset there, made to hold the copies below it, which stay\./, 'a copy onto a holder says it replaces it'); +assert.doesNotMatch(context.zfsmReplicatePlanHtml({can_proceed: true, mode: 'full', size: null, holder: false}), /replaces the empty dataset/, 'only onto a holder'); assert.match(context.zfsmReplicatePlanHtml({can_proceed: true, mode: 'incremental', since: 'a[\s\S]*Sent raw/, 'a size not yet known is left out; an encrypted dataset goes raw'); assert.match(context.zfsmReplicatePlanHtml({can_proceed: true, mode: 'resume', size: 0, raw: false}), /Carries on from where the last run to this copy stopped, about 0 B\./); assert.match(context.zfsmReplicatePlanHtml({can_proceed: true, mode: 'full', size: 1, raw: false, type: 'filesystem'}), /The copy is unmounted and read-only/, 'a filesystem\'s copy is unmounted'); diff --git a/tests/replicate_test.php b/tests/replicate_test.php index f55cac20..ab265bba 100644 --- a/tests/replicate_test.php +++ b/tests/replicate_test.php @@ -59,7 +59,7 @@ function same($expected, $actual, $message) { same(0, zfsmRunReplicate('tank/data', 'b', 'backup/data'), 'it runs'); same(array('send -P -v -L tank/data@b'), $calls('send'), 'send streams the snapshot itself, with progress and -L'); same(array('send -P -v -n -L tank/data@b'), array_values(array_unique(preg_grep('/^send -P -v -n /', $read('calls.log')))), 'each estimate counts the same stream'); -same(array('receive -s -u -o readonly=on -o canmount=noauto backup/data'), $calls('receive'), 'receive keeps it resumable, unmounted and read-only, never -F'); +same(array('receive -s -u -o readonly=on -o canmount=noauto backup/data'), $calls('receive'), 'receive keeps it resumable, unmounted and read-only, with no -F into a new copy'); same(array('backup/data@b'), $names('backup/data', 'snapshot'), 'the copy holds b'); same(array($bookmark('backup/data', 'b')), $names('tank/data', 'bookmark'), 'and the dataset a bookmark for this destination at b'); same(array('replicate_start', 'replicate_stop'), array_values(array_unique($GLOBALS['zfsm_feed'])), 'the row shows the run'); diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 75a53c27..07eb09db 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -153,7 +153,7 @@ function same($expected, $actual, $message) { same(array('normal: Scheduled copies under backup: 6 of 6 dataset(s) copied. It works again after 1 run(s) with a problem.'), $read('notify-detail.log'), 'and says so'); same(array("tank/data@$n" => $tag), $snaps('tank/data'), 'its snapshot is tagged for the record'); same(1, $logged('/ Schedule '.$id.': Scheduled copies under backup: 6 of 6 dataset\(s\) copied$/'), 'logged'); -same(array(), preg_grep('/receive .*-F/', $read('calls.log')), 'never with -F'); +same(array(), preg_grep('/receive .*-F/', $read('calls.log')), 'no -F, as no holder is replaced'); // The one after sends changes; Keep leaves the newest 1 of the record's snapshots on each dataset. $reset(); @@ -171,14 +171,51 @@ function same($expected, $actual, $message) { same(0, zfsmScheduleRun($id, $cfg, $t + 9000), 'a run with a copy gone'); same(array(true, 1), array(in_array('backup/usb', $datasets(), true), count(preg_grep('/^send -P -v -L usb@/', $read('calls.log')))), 'makes it again in full'); -// tank/san no longer Sanoid's: its copy's place holds a holder, which it can't replace; tank/san/kid goes on. +// tank/san no longer Sanoid's: its first copy replaces the holder in its place (receive -F), and the copy below stays; the copy loses +// the holder's mark, which the one below inherited. $pool('marks', array_values(array_filter($read('pool/marks'), function($line) { return strpos($line, 'tank/san@autosnap_') !== 0; }))); $reset(); -same(1, zfsmScheduleRun($id, $cfg, $t + 10800), 'a run with the holder in the way'); +same(0, zfsmScheduleRun($id, $cfg, $t + 10800), 'a run with a holder in a copy\'s place'); $o = $name($t + 10800); -same(1, preg_match('/^warning: Scheduled copies under backup: 6 of 7 dataset\(s\) copied\. Replication of tank\/san to backup\/tank\/san not started: backup\/tank\/san only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them/', $read('notify-detail.log')[0] ?? ''), 'says so'); -same(array(), $snaps('tank/san'), 'the snapshot it took for it goes'); -same(array("backup/tank/san/kid@$o"), array_keys($snaps('backup/tank/san/kid')), 'the one below is copied'); +same(array(1, array("backup/tank/san@$o")), array(count(preg_grep('/^receive -s -u -o readonly=on -F -o canmount=noauto backup\/tank\/san$/', $read('calls.log'))), array_keys($snaps('backup/tank/san'))), 'its copy replaces the holder'); +same(array(false, false), array(zfsmReplicaHolder(array('server' => ''), 'backup/tank/san'), zfsmReplicaHolder(array('server' => ''), 'backup/tank/san/kid')), 'the mark is cleared'); +same(array(1, array("backup/tank/san/kid@$o")), array(count(preg_grep('/^send -P -v -L -i \S+ tank\/san\/kid@/', $read('calls.log'))), array_keys($snaps('backup/tank/san/kid'))), 'the one below stays: it gets only the changes'); +same(1, count(preg_grep('/^receive .*-F/', $read('calls.log'))), '-F only onto the holder'); + +// An encrypted dataset's raw copy can't replace a holder: zfs refuses it there, so the plan says what to do instead. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/enc\tfilesystem", "backup/tank/enc\tfilesystem"))); +$pool('marks', array_merge($read('pool/marks'), array("tank/enc@a\t4242\t9000"))); +$pool('props', array_merge($read('pool/props'), array("tank/enc\tencryption\taes-256-gcm", "backup/tank/enc\tzfs.master:holder\ton"))); +same(array('can_proceed' => false, 'error' => 'backup/tank/enc only holds the copies of datasets below this one, and zfs can\'t receive this encrypted dataset\'s raw copy over it: set its Replication to None under Actions, Schedules, or destroy that dataset and the copies below it for the next run to send them all'), + zfsmReplicatePlan('tank/enc', 'a', 'backup/tank/enc', array('estimate' => false)), 'a raw stream onto a holder'); + +// A first copy onto a holder that stops part-way carries on only with -F again (OpenZFS 2.4.4); the mark goes once it lands. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/res\tfilesystem", "backup/tank/res\tfilesystem"))); +$pool('marks', array_merge($read('pool/marks'), array("tank/res@a\t4343\t9001"))); +$pool('props', array_merge($read('pool/props'), array("backup/tank/res\tzfs.master:holder\ton"))); +$reset(); +putenv('ZFSM_TEST_SEND_CUT=1'); +same(1, zfsmRunReplicate('tank/res', 'a', 'backup/tank/res'), 'a first copy onto a holder, cut off'); +putenv('ZFSM_TEST_SEND_CUT'); +same(array(true, 'resume'), array(zfsmReplicaHolder(array('server' => ''), 'backup/tank/res'), zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false))['mode']), 'still a holder, part-received'); +same(0, zfsmRunReplicate('tank/res', 'a', 'backup/tank/res'), 'carried on'); +same(array(2, array('backup/tank/res@a'), false), array(count(preg_grep('/^receive -s -u -o readonly=on -F /', $read('calls.log'))), array_keys($snaps('backup/tank/res')), zfsmReplicaHolder(array('server' => ''), 'backup/tank/res')), 'with -F both times, landed, the mark gone'); + +// A mark that could not be cleared is told; the copy holds a snapshot then, so a later run of changes cut off carries on without -F. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/mark\tfilesystem", "backup/tank/mark\tfilesystem"))); +$pool('marks', array_merge($read('pool/marks'), array("tank/mark@a\t4444\t9002", "tank/mark@b\t4445\t9003"))); +$pool('props', array_merge($read('pool/props'), array("backup/tank/mark\tzfs.master:holder\ton"))); +$reset(); +putenv('ZFSM_TEST_INHERIT_FAIL=1'); +same(0, zfsmRunReplicate('tank/mark', 'a', 'backup/tank/mark'), 'a first copy onto a holder whose mark stays'); +putenv('ZFSM_TEST_INHERIT_FAIL'); +same(array('warning: Replication of tank/mark@a to backup/tank/mark finished (in full). The copy still carries the mark of the empty dataset it replaced: cannot inherit zfs.master:holder for ?backup/tank/mark?: permission denied'), $read('notify-detail.log'), 'says so'); +$reset(); +putenv('ZFSM_TEST_SEND_CUT=1'); +same(1, zfsmRunReplicate('tank/mark', 'b', 'backup/tank/mark'), 'its changes, cut off'); +putenv('ZFSM_TEST_SEND_CUT'); +same(array('resume', false), array_values(array_intersect_key(zfsmReplicatePlan('tank/mark', 'b', 'backup/tank/mark', array('estimate' => false)), array('mode' => 0, 'holder' => 0))), 'carried on as changes, not onto a holder'); +same(array(0, array(), array('backup/tank/mark@a', 'backup/tank/mark@b')), array(zfsmRunReplicate('tank/mark', 'b', 'backup/tank/mark'), preg_grep('/^receive .*-F/', $read('calls.log')), array_keys($snaps('backup/tank/mark'))), 'without -F'); // One cut off part-way holds for each copy, as a run by hand does: carried on by hand, its holds come off. putenv('ZFSM_TEST_SEND_CUT=1'); diff --git a/tests/schedule_snapshot_test.php b/tests/schedule_snapshot_test.php index 9f0975ba..688c001a 100644 --- a/tests/schedule_snapshot_test.php +++ b/tests/schedule_snapshot_test.php @@ -358,12 +358,12 @@ function same($expected, $actual, $message) { array($own(), count($stored()), is_file(zfsmScheduleStateFile('5a5a5a5a')), is_file(zfsmScheduleStateFile('5b5b5b5b'))), 'its own schedules and those below follow it'); $inode = fileinode(zfsmScheduleFile()); same(array(array('succeeded' => array('tank/vm2' => 0), 'failed' => array()), $inode), array(renameDataset('tank/vm', 'tank/vm2', 0), fileinode(zfsmScheduleFile())), 'one with none writes nothing'); -$pool('datasets', array("tank\tfilesystem", "tank/new\tfilesystem", "tank/new/sub\tfilesystem", "tank/docker\tfilesystem")); $lock = zfsmJobLock(zfsmJobDir('schedule').'/settings.lock', false); same(array('succeeded' => array('tank/newer' => 0), 'failed' => array('Snapshot schedules of tank/new' => 'not moved to tank/newer (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules')), renameDataset('tank/new', 'tank/newer', 0), 'one whose schedules could not be moved says so'); zfsmJobUnlock($lock); // Destroy: they go, counts too. +$pool('datasets', array("tank\tfilesystem", "tank/new\tfilesystem", "tank/new/sub\tfilesystem", "tank/docker\tfilesystem")); touch(zfsmScheduleStateFile($ids['tank/data'])); same(array('succeeded' => array('tank/new' => 0), 'failed' => array()), destroyDataset('tank/new', 1), 'a destroy'); same(array(array('tank/docker' => $ids['tank/docker']), false), array($own(), is_file(zfsmScheduleStateFile($ids['tank/data']))), 'its own schedules and those below go'); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index 42289789..057969d5 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -347,7 +347,7 @@ _(Replicate Every Dataset)_: > > - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full, as does a run after a copy is gone, such as when the pool it goes into was made again. The copies are read-only and not mounted. > - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy, one whose name is too long for the bookmark its copy keeps (over about 70 characters under `backup`, fewer under a longer **Copy Under**), and, when a run can't read the snapshot names, those it couldn't read: they may be left to Sanoid or Syncoid. -> - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. When the copy of the dataset above one is missing because it failed, or its own copy schedule has not made it yet, that one waits for the next run. +> - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. If that dataset is copied later, its first copy replaces the empty one and the copies below it stay; an encrypted dataset sent raw can't replace it, and the run says what to do. When the copy of the dataset above one is missing because it failed, or its own copy schedule has not made it yet, that one waits for the next run. > - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. > - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. A run that falls due while the one before still goes, as a first run can, is skipped and logged; once the one before has gone on for a day, it counts as a problem. diff --git a/zfs.master/frontend/ZFSMImpact.js b/zfs.master/frontend/ZFSMImpact.js index 1db1d91f..13d9bf76 100644 --- a/zfs.master/frontend/ZFSMImpact.js +++ b/zfs.master/frontend/ZFSMImpact.js @@ -181,6 +181,7 @@ function zfsmReplicatePlanHtml(plan) { : plan.mode === 'incremental' ? 'Sends only the changes since '+esc(plan.since)+size+'.' : 'The first replication sends everything'+size+'.'); if (plan.raw) html += say('Sent raw: the copy stays locked with this dataset\'s key.'); + if (plan.holder) html += say('It replaces the empty dataset there, made to hold the copies below it, which stay.'); html = area+html; // An incremental or resumed receive into a copy here that has a /dev/zvol device removes it (volmode=none, zfsmReplicateReceiveArgs); // one on another server never had one (zfsmReplicaAreaProps). diff --git a/zfs.master/include/ZFSMReplicaPlan.php b/zfs.master/include/ZFSMReplicaPlan.php index 64bfa447..af8f649a 100644 --- a/zfs.master/include/ZFSMReplicaPlan.php +++ b/zfs.master/include/ZFSMReplicaPlan.php @@ -61,9 +61,10 @@ function zfsmReplicateSendArgs($plan, $dry_run = false) { } // -s: a stopped run resumes. -u and canmount=noauto keep a filesystem's copy unmounted, even after a reboot; volmode=none keeps a volume's -// copy from being a second block device of the same data. Resumed and incremental receives apply them too. +// copy from being a second block device of the same data. Resumed and incremental receives apply them too. -F only onto a holder. function zfsmReplicateReceiveArgs($plan) { $args = array('receive', '-s', '-u', '-o', 'readonly=on'); + if ($plan['holder']) $args[] = '-F'; array_push($args, '-o', $plan['type'] === 'filesystem' ? 'canmount=noauto' : 'volmode=none'); $args[] = $plan['dest']; return $args; @@ -79,6 +80,8 @@ function zfsmReplicateEstimate($plan) { return ($estimate['code'] ?? 1) === 0 ? zfsmReplicateSize($estimate['output']) : null; } +define('ZFSM_REPLICA_TOO_LONG', 'The names are too long for the bookmark that keeps the next replication incremental'); + function zfsmReplicaBookmarkFits($zdataset, $mark, $snapshot) { return strlen($zdataset.'#'.zfsmReplicaBookmarkPrefix($mark).$snapshot) <= 255; } @@ -88,7 +91,6 @@ function zfsmReplicaBookmarkFits($zdataset, $mark, $snapshot) { // $with['marks'] passes the dataset's marks already read; $with['estimate'] false skips the zfs send -n; $with['target'] is where the copy lives (zfsmReplicaTargetDest). function zfsmReplicatePlan($zdataset, $snapshot, $zdest, $with = array()) { $refuse = function($why) { return array('can_proceed' => false, 'error' => $why); }; - $too_long = 'The names are too long for the bookmark that keeps the next replication incremental'; $target = $with['target'] ?? array('server' => ''); if (isset($target['error'])) return $refuse($target['error']); if (!isset($with['marks']) && !zfsmDatasetExists($zdataset)) return $refuse('Only an existing dataset can be replicated'); @@ -108,37 +110,19 @@ function zfsmReplicatePlan($zdataset, $snapshot, $zdest, $with = array()) { list($missing, $why) = zfsmReplicaTargetMissing($target, $zdest); if ($missing === null) return $refuse('Could not check whether '.$where.' exists'.$because($why)); $key = zfsmReplicaTargetMark($target, $zdest); - $plan = array('can_proceed' => true, 'mode' => 'full', 'snapshot' => $from, 'dest' => $zdest, 'server' => $target['server'], 'mark' => $key, 'raw' => $encryption !== 'off', 'type' => $type, 'base' => null, 'token' => null); + $plan = array('can_proceed' => true, 'mode' => 'full', 'snapshot' => $from, 'dest' => $zdest, 'server' => $target['server'], 'mark' => $key, 'raw' => $encryption !== 'off', 'type' => $type, 'base' => null, 'token' => null, 'holder' => false); list($token, $why) = $missing ? array('', '') : zfsmReplicaTargetToken($target, $zdest); if ($token === null) return $refuse('Could not check whether '.$where.' holds a part-received replication'.$because($why)); if ($token !== '') { - // A run that stopped part-way left a token on the copy: zfs send -t carries on from there, whatever snapshot is picked now. - $info = runProcess('zfs', array('send', '-P', '-v', '-n', '-t', $token)); - if (($info['code'] ?? 1) !== 0 || !preg_match('/^\s*toname = (.+?)\s*$/m', (string)$info['output'], $match)) { - return $refuse($where.' holds a part-received replication that can\'t be resumed ('.zfsmOneLine($info['output'] ?? '').'); zfs receive -A '.$zdest.' discards it'.($target['server'] === '' ? '' : ' there')); - } - if (strncmp($match[1], $zdataset.'@', strlen($zdataset) + 1) !== 0) return $refuse($where.' holds a part-received replication of '.$match[1].', not of this dataset'); - if (!zfsmReplicaBookmarkFits($zdataset, $key, substr($match[1], strlen($zdataset) + 1))) return $refuse($too_long); - return array_merge($plan, array('mode' => 'resume', 'snapshot' => $match[1], 'token' => $token, 'size' => zfsmReplicateSize($info['output']))); + $plan = zfsmReplicaPlanResume($plan, $target, $zdataset, $token); + return is_string($plan) ? $refuse($plan) : $plan; } $snap = $from === null ? array('name' => null, 'guid' => '', 'txg' => PHP_INT_MAX) : null; foreach ($marks as $mark) if ($from !== null && $mark['name'] === $from) $snap = $mark; if ($snap === null) return $refuse('The snapshot '.$from.' no longer exists'); - if (!zfsmReplicaBookmarkFits($zdataset, $key, $from !== null ? $snapshot : ($with['new_name'] ?? ''))) return $refuse($too_long); - if (!$missing) { - list($held, $why) = zfsmReplicaTargetSnapshots($target, $zdest); - if ($held === null) return $refuse('The snapshots of '.$where.' could not be read'.$because($why)); - if (!$held && zfsmReplicaHolder($target, $zdest)) return $refuse($where.' only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them'); - if (!$held) return $refuse($where.' already exists and holds no snapshot, so it can\'t be updated without destroying it'); - $newest = end($held); - $base = null; - // The snapshot or bookmark the copy's newest snapshot came from; a snapshot first, as older OpenZFS can't send raw from some bookmarks. - foreach ($marks as $mark) if ($mark['guid'] === $newest['guid'] && ($base === null || strpos($mark['name'], '@') !== false)) $base = $mark; - if ($base === null) return $refuse($newest['name'].' is not on this dataset, so '.$where.' can\'t be updated without destroying it'); - if ($base['guid'] === $snap['guid']) return $refuse($where.' already holds '.$from); - if ($base['txg'] > $snap['txg']) return $refuse($where.' already holds '.$base['name'].', which is newer than '.$from); - $plan = array_merge($plan, array('mode' => 'incremental', 'base' => $base['name'], 'since' => zfsmReplicaShortName($zdataset, $base['name']))); - } + if (!zfsmReplicaBookmarkFits($zdataset, $key, $from !== null ? $snapshot : ($with['new_name'] ?? ''))) return $refuse(ZFSM_REPLICA_TOO_LONG); + if (!$missing) $plan = zfsmReplicaPlanOnto($plan, $target, $zdataset, $marks, $snap); + if (is_string($plan)) return $refuse($plan); // A bookmark for this copy means a run made it before: a schedule's full send there would refill what someone removed. if ($missing && ($with['scheduled'] ?? false)) { $prefix = $zdataset.'#'.zfsmReplicaBookmarkPrefix($key); @@ -147,6 +131,47 @@ function zfsmReplicatePlan($zdataset, $snapshot, $zdest, $with = array()) { return array_merge($plan, array('size' => $from === null || ($with['estimate'] ?? true) === false ? null : zfsmReplicateEstimate($plan))); } +// $plan carrying on the run that stopped part-way and left $token on the copy (zfs send -t), whatever snapshot is picked now; onto +// a holder with -F again, which zfs asks for there (live, 2.4.4). Else why not. +function zfsmReplicaPlanResume($plan, $target, $zdataset, $token) { + $zdest = $plan['dest']; + $where = zfsmReplicaTargetWhere($target, $zdest); + $info = runProcess('zfs', array('send', '-P', '-v', '-n', '-t', $token)); + if (($info['code'] ?? 1) !== 0 || !preg_match('/^\s*toname = (.+?)\s*$/m', (string)$info['output'], $match)) { + return $where.' holds a part-received replication that can\'t be resumed ('.zfsmOneLine($info['output'] ?? '').'); zfs receive -A '.$zdest.' discards it'.($target['server'] === '' ? '' : ' there'); + } + if (strncmp($match[1], $zdataset.'@', strlen($zdataset) + 1) !== 0) return $where.' holds a part-received replication of '.$match[1].', not of this dataset'; + if (!zfsmReplicaBookmarkFits($zdataset, $plan['mark'], substr($match[1], strlen($zdataset) + 1))) return ZFSM_REPLICA_TOO_LONG; + $holder = zfsmReplicaHolder($target, $zdest) && zfsmReplicaTargetSnapshots($target, $zdest)[0] === array(); + return array_merge($plan, array('mode' => 'resume', 'snapshot' => $match[1], 'token' => $token, 'size' => zfsmReplicateSize($info['output']), 'holder' => $holder)); +} + +// $plan into a copy that is there: the changes since the snapshot or bookmark its newest snapshot came from, else in full onto a holder +// (zfsmReplicaPlanHolder); or why not. $snap: the mark it sends. +function zfsmReplicaPlanOnto($plan, $target, $zdataset, $marks, $snap) { + $where = zfsmReplicaTargetWhere($target, $plan['dest']); + list($held, $why) = zfsmReplicaTargetSnapshots($target, $plan['dest']); + if ($held === null) return 'The snapshots of '.$where.' could not be read'.($why !== '' ? ': '.$why : ''); + if (!$held) return zfsmReplicaPlanHolder($plan, $target); + $newest = end($held); + $base = null; + // The snapshot or bookmark the copy's newest snapshot came from; a snapshot first, as older OpenZFS can't send raw from some bookmarks. + foreach ($marks as $mark) if ($mark['guid'] === $newest['guid'] && ($base === null || strpos($mark['name'], '@') !== false)) $base = $mark; + if ($base === null) return $newest['name'].' is not on this dataset, so '.$where.' can\'t be updated without destroying it'; + if ($base['guid'] === $snap['guid']) return $where.' already holds '.$snap['name']; + if ($base['txg'] > $snap['txg']) return $where.' already holds '.$base['name'].', which is newer than '.$snap['name']; + return array_merge($plan, array('mode' => 'incremental', 'base' => $base['name'], 'since' => zfsmReplicaShortName($zdataset, $base['name']))); +} + +// $plan into a copy that holds no snapshot: in full with -F when it is a holder, which keeps the copies below it (live, 2.4.4); zfs +// refuses a raw stream there. Else why not. +function zfsmReplicaPlanHolder($plan, $target) { + $where = zfsmReplicaTargetWhere($target, $plan['dest']); + if (!zfsmReplicaHolder($target, $plan['dest'])) return $where.' already exists and holds no snapshot, so it can\'t be updated without destroying it'; + if ($plan['raw']) return $where.' only holds the copies of datasets below this one, and zfs can\'t receive this encrypted dataset\'s raw copy over it: set its Replication to None under Actions, Schedules, or destroy that dataset and the copies below it for the next run to send them all'; + return array_merge($plan, array('holder' => true)); +} + // What the Replicate dialog shows: the snapshots to pick from, newest first, the last destination, the trusted servers, and the plan // when a destination is given; for a trusted server ($server) also its area, or why it can't be reached. function zfsmReplicatePreview($zdataset, $snapshot, $zdest, $cfg, $server = '') { diff --git a/zfs.master/include/ZFSMReplicaTarget.php b/zfs.master/include/ZFSMReplicaTarget.php index de111fa3..bf3c301a 100644 --- a/zfs.master/include/ZFSMReplicaTarget.php +++ b/zfs.master/include/ZFSMReplicaTarget.php @@ -36,6 +36,12 @@ function zfsmReplicaHolderMake($zdest) { return ($made['code'] ?? 1) === 0 ? '' : 'The dataset to hold the copies below '.$zdest.' could not be made: '.zfsmCommandError($made, 'zfs create failed'); } +// After a copy landed in a holder's place (zfsmReplicaPlanHolder), which keeps the holder's mark: clears it. '' or why not. +function zfsmReplicaHolderClear($zdest) { + $cleared = runProcess('zfs', array('inherit', ZFSM_REPLICA_HOLDER, $zdest)); + return ($cleared['code'] ?? 1) === 0 ? '' : 'The copy still carries the mark of the empty dataset it replaced: '.zfsmCommandError($cleared, 'zfs inherit failed'); +} + // Whether $zdest on $target is a holder; false on a trusted server, which makes none, and when it can't be read. function zfsmReplicaHolder($target, $zdest) { if ($target['server'] !== '') return false; diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index 589b7830..9e9277af 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -1,6 +1,6 @@ Date: Wed, 18 Nov 2026 09:17:32 +0800 Subject: [PATCH 02/24] feat(schedules): Schedules sets a dataset's own copies under a dataset, and a rename takes its copies along Main's Schedules dialog gains Replication: Default, None or its own repeat with Copy Under and Keep, saved with its snapshot schedule in one change. A rename moves its copies under each Copy Under onto a free name, after holders; a destroy leaves them. --- .coderabbit.yaml | 1 + README.md | 2 +- tests/admin_route_test.php | 16 +- tests/diagnostics_bundle_test.php | 7 +- tests/fixtures/bin/zfs | 2 + tests/page_dialog_test.cjs | 31 +++- tests/phone_render_test.cjs | 8 +- tests/run_http_inventory_tests.sh | 3 +- tests/schedule_dialog_test.php | 170 ++++++++++++++++++++ tests/schedule_snapshot_test.php | 5 +- zfs.master/ZFSMaster.page | 6 +- zfs.master/ZFSMasterSettings.page | 4 +- zfs.master/backend/ZFSMAdmin.php | 30 ++-- zfs.master/backend/ZFSMOperations.php | 4 +- zfs.master/frontend/ZFSMImpact.js | 39 +++-- zfs.master/include/ZFSMSchedule.php | 1 + zfs.master/include/ZFSMScheduleCopies.php | 114 +++++++++++-- zfs.master/include/ZFSMScheduleDialog.php | 64 ++++++++ zfs.master/include/ZFSMScheduleSnapshot.php | 52 +++--- 19 files changed, 476 insertions(+), 83 deletions(-) create mode 100644 tests/schedule_dialog_test.php create mode 100644 zfs.master/include/ZFSMScheduleDialog.php diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 97a16232..e86a603f 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -40,6 +40,7 @@ reviews: - File names, ZFS user properties and anything from another server may not be valid UTF-8. htmlspecialchars() needs ENT_SUBSTITUTE, or it returns '' and the text disappears. json_encode() needs JSON_INVALID_UTF8_SUBSTITUTE, or its false result must be handled; never echo, publish or hash a json_encode() that can fail. - Every `zfs receive` takes `-s -u -o readonly=on`, and `-F` only onto a holder that holds no snapshot (zfsmReplicaPlanHolder, zfsmReplicaPlanResume), which zfs needs to replace it. A copy of a volume must not appear as a block device (volmode=none). - Replicate Every Dataset (include/ZFSMScheduleCopies.php) makes an empty holder (`zfs create -o canmount=off -o readonly=on -o zfs.master:holder=on`) only where a copy's parent is missing. The property inherits, so a dataset is a holder only where its source is `local` (zfsmReplicaHolder). + - A rename moves the dataset's copies under each Copy Under (zfsmScheduleCopiesMove) only onto a free name, after holders for missing parents; a destroy leaves its copies, by design. - path: "zfs.master/{include/ZFSMDiag*.php,scripts/diagnostics.php}" instructions: | Diagnostics: a zip the owner may post in a public issue. Its privacy scope is fixed; report gaps inside it and nothing past it. diff --git a/README.md b/README.md index 7d8aef03..c362af89 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ See and manage your ZFS pools, datasets and snapshots from Unraid's **Main** tab Each dataset row has an **Actions** menu. These items do more than their name says: - **Replicate** — copies the dataset to another pool, or to another ZFS Master NG server added in Settings. Run it once, or repeat it hourly or on chosen days and keep the newest snapshots, plus the newest of each of the last days, weeks, months or years you choose. New sends keep records larger than 128K whole; what a copy already holds stays as it is. A copy of an unencrypted dataset to a server travels compressed with zstd when both servers have it. -- **Schedules** — gives the dataset its own snapshot schedule, or None, in place of Snapshot Every Dataset (Settings). Datasets below it follow it unless they have their own. +- **Schedules** — gives the dataset its own snapshot schedule, and its own copies into another pool (Replication), or None, in place of Snapshot Every Dataset and Replicate Every Dataset (Settings). Datasets below it follow it unless they have their own. Renaming the dataset renames its copies under Copy Under too. - **Rewrite Files** (Unraid 7.3+) — writes the dataset's files again, so a changed compression, checksum, dedup or copies setting applies to data already there. Snapshots keep the old blocks, so the pool can use more space until they are deleted. - **Snapshots Admin** — lists the dataset's snapshots, and its children's if you like, to filter and sort. From there you can browse a snapshot's files, see what changed since it, roll back, clone, hold, rename, or delete several at once. - **Promote Dataset** (clones only) — makes a clone stand on its own, no longer tied to the snapshot it was cloned from. The original dataset can then be destroyed and the clone stays. diff --git a/tests/admin_route_test.php b/tests/admin_route_test.php index 80cc55e0..cb107df6 100644 --- a/tests/admin_route_test.php +++ b/tests/admin_route_test.php @@ -32,10 +32,14 @@ 'schedulesnapshotsown' => array('cmd' => 'schedulesnapshots', 'zdataset' => 'tank/data', 'every' => 'days', 'time' => '04:00', 'days' => '1', 'keep' => ''), 'schedulecopies' => array('cmd' => 'schedulecopies', 'under' => 'backup', 'every' => 'hourly', 'time' => '', 'days' => '', 'keep' => '3'), 'schedulecopiesnowhere' => array('cmd' => 'schedulecopies', 'under' => 'nowhere', 'every' => 'hourly', 'time' => '', 'days' => '', 'keep' => ''), + 'scheduledataset' => array('cmd' => 'scheduledataset', 'zdataset' => 'tank/data', 'every' => 'days', 'time' => '04:00', 'days' => '1', 'keep' => '', + 'copy_every' => 'hourly', 'copy_time' => '', 'copy_days' => '', 'copy_keep' => ' 24 7D ', 'under' => 'backup'), + 'scheduledatasetwrong' => array('cmd' => 'scheduledataset', 'zdataset' => 'tank/data', 'every' => 'days', 'time' => '', 'days' => '', 'keep' => '', + 'copy_every' => 'hourly', 'copy_time' => '', 'copy_days' => '', 'copy_keep' => 'x5', 'under' => ''), 'previewschedules' => array('cmd' => 'previewschedules', 'zdataset' => 'tank/data'), 'previewschedulesbad' => array('cmd' => 'previewschedules', 'zdataset' => 'tank/bad name!'), ); -$replication = array('previewreplicate', 'replicatedataset', 'schedulepicked', 'schedulenowhere', 'schedulekeep', 'schedulesnapshots', 'schedulesnapshotstiers', 'schedulesnapshotswrong', 'schedulesnapshotsown', 'schedulecopies', 'schedulecopiesnowhere', 'previewschedules', 'previewschedulesbad'); +$replication = array('previewreplicate', 'replicatedataset', 'schedulepicked', 'schedulenowhere', 'schedulekeep', 'schedulesnapshots', 'schedulesnapshotstiers', 'schedulesnapshotswrong', 'schedulesnapshotsown', 'schedulecopies', 'schedulecopiesnowhere', 'scheduledataset', 'scheduledatasetwrong', 'previewschedules', 'previewschedulesbad'); $route = $argv[1] ?? ''; if (!isset($routes[$route])) exit(2); @@ -61,7 +65,7 @@ // A schedule to remove, or a snapshot schedule saved (the default or a dataset's own), on the tmpfs the runner mounts on /boot/config. No update_cron here, so // Unraid's crontab is not rebuilt. -if ($route === 'scheduleremovesaved' || $route === 'schedulesnapshots' || $route === 'schedulesnapshotstiers' || $route === 'schedulesnapshotsown' || $route === 'schedulecopies') { +if (in_array($route, array('scheduleremovesaved', 'schedulesnapshots', 'schedulesnapshotstiers', 'schedulesnapshotsown', 'schedulecopies', 'scheduledataset'), true)) { zfsmTestScratch('/boot/config'); putenv('ZFSM_TEST_JOB_DIR='.sys_get_temp_dir().'/zfsm-route-jobs-'.getmypid()); @mkdir('/boot/config/plugins/zfs.master', 0700, true); @@ -120,7 +124,13 @@ && strpos($answer['schedules_html'] ?? '', '>Replicate Every Dataset under backup, every hour at :') !== false && array_column(json_decode(file_get_contents('/boot/config/plugins/zfs.master/schedules.json'), true), 'dest') === array('backup/'), 'schedulecopiesnowhere' => $answer === array('error' => 'Nothing was saved: nowhere does not exist on this server') && !is_file('/boot/config/plugins/zfs.master/schedules.json'), - 'previewschedules' => $answer === array('every' => 'default', 'time' => '00:00', 'days' => range(0, 6), 'keep' => '', 'inherited' => 'Snapshot Every Dataset, which is off: no scheduled snapshots', 'skipped' => ''), + 'scheduledataset' => preg_match('/^Saved: a snapshot of each of 1 dataset\(s\) \(tank\/data and below\) every Monday at 04:00; it keeps all it takes\. Saved: a copy of each of 1 dataset\(s\) \(tank\/data and below\) under backup every hour at :[0-5][0-9]; it keeps the newest 24 and 7 daily of the snapshots of each dataset it copies$/D', $answer['said'] ?? '') === 1 + && strpos($answer['schedules_html'] ?? '', '>Copies of tank/data and below under backup, every hour at :') !== false + && array_map(null, ...array_map(function($key) { return array_column(json_decode(file_get_contents('/boot/config/plugins/zfs.master/schedules.json'), true), $key); }, array('dest', 'keep'))) === array(array('', 0), array('backup/', '24 7d')), + 'scheduledatasetwrong' => $answer === array('error' => 'Nothing was saved: Snapshots: set the time it starts at; tick at least one day; Replication: under Copy Under, name a dataset on this server, such as backup') + && !is_file('/boot/config/plugins/zfs.master/schedules.json'), + 'previewschedules' => $answer === array('every' => 'default', 'time' => '00:00', 'days' => range(0, 6), 'keep' => '', 'inherited' => 'Snapshot Every Dataset, which is off: no scheduled snapshots', 'skipped' => '', + 'copies' => array('every' => 'default', 'time' => '00:00', 'days' => range(0, 6), 'keep' => '', 'under' => '', 'inherited' => 'Replicate Every Dataset, which is off: no scheduled copies', 'skipped' => '')), 'previewschedulesbad' => ($answer['failed'] ?? null) === array('request' => 'Invalid dataset name'), )[$route]; if (!$ok) { diff --git a/tests/diagnostics_bundle_test.php b/tests/diagnostics_bundle_test.php index 6a83397e..a6fde18c 100644 --- a/tests/diagnostics_bundle_test.php +++ b/tests/diagnostics_bundle_test.php @@ -134,7 +134,8 @@ function has($needle, $text, $message) { 'Schedule 0a1b2c40: Scheduled copies under tank/CanaryUnder: 1 of 8 dataset(s) copied. Not copied, as the copy of a dataset above it is missing (tank/CanaryWaitA, tank/CanaryWait B, tank/CanaryWaitC, and 2 more). ' ."The dataset to hold the copies below tank/CanaryUnder/tank/CanaryHold could not be made: cannot create 'tank/CanaryUnder/tank/CanaryHold': out of space", 'Replication of tank/CanaryCpSrc to tank/CanaryUnder/tank/CanaryCpSrc not started: tank/CanaryUnder/tank/CanaryCpSrc only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them', - 'ZFS Dataset Rename: Snapshot schedules of tank/CanaryMvFrom failed: not moved to tank/CanaryMvTo (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules', + 'ZFS Dataset Rename: Schedules of tank/CanaryMvFrom failed: not moved to tank/CanaryMvTo (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules', + "ZFS Dataset Rename: Copy tank/CanaryUnder/tank/CanaryMvCopy failed: not renamed with it: cannot rename 'tank/CanaryUnder/tank/CanaryMvCopy': dataset is busy. It stays under its old name, so the next copy of the renamed dataset is a full one", // A schedule's Keep: what it removed, what stayed and why, and holds a run could not put on or take off. 'Removed tank/CanaryDs One@CanaryKeepOld (not among the newest 3, 7 daily and 4 weekly its schedule keeps)', // Snapshots Prefix can put any words in a snapshot's name, Keep's own among them. @@ -317,8 +318,10 @@ function has($needle, $text, $message) { has('Schedule '.zfsmDiagToken('0a1b2c40').": Scheduled copies under $under: 1 of 8 dataset(s) copied. Not copied, as the copy of a dataset above it is missing (tank/".$token('CanaryWaitA').', tank/'.$token('CanaryWait B').', tank/'.$token('CanaryWaitC').', and 2 more). ' ."The dataset to hold the copies below $hold could not be made: cannot create ?$hold?: out of space\n", $log, 'a run of copies under a dataset: each name a token, the plugin\'s words in clear'); has('Replication of tank/'.$token('CanaryCpSrc')." to $cp not started: $cp only holds the copies of datasets below this one, made while it was not copied; its own copy cannot go there without destroying them\n", $log, 'and one in a copy\'s place'); -has('ZFS Dataset Rename: Snapshot schedules of tank/'.$token('CanaryMvFrom').' failed: not moved to tank/'.$token('CanaryMvTo').' (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules'."\n", +has('ZFS Dataset Rename: Schedules of tank/'.$token('CanaryMvFrom').' failed: not moved to tank/'.$token('CanaryMvTo').' (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules'."\n", $log, 'a rename whose schedules were not moved'); +$mv = $under.'/'.zfsmDiagToken('tank').'/'.$token('CanaryMvCopy'); +has("ZFS Dataset Rename: Copy $mv failed: not renamed with it: cannot rename ?$mv?: dataset is busy. It stays under its old name, so the next copy of the renamed dataset is a full one\n", $log, 'and a copy left behind'); // Keep Replicas on a copy received here, which the next receive tries again. $in = 'tank/'.$ds.'/'.zfsmDiagToken('canarysender').'/'; has('Removed '.$in.$token('CanaryIn').'@'.zfsmDiagToken('CanaryRecvOld')." (not among the newest 2 Keep Replicas keeps)\n", $log, 'a removal here keeps its why'); diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index a4fa1c38..673aea8a 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -85,7 +85,9 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ]; then exists "$last_arg" && { awk -F'\t' -v n="$last_arg" '$1 == n { print $1 "\t" $2 }' "$R/datasets"; exit 0; } printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1 fi + # A name's read; ZFSM_TEST_LIST_FAIL names one whose read fails. if [ "$2" = -H ] && [ "$3" = -o ] && [ "$4" = name ]; then + [ "${ZFSM_TEST_LIST_FAIL:-}" = "$last_arg" ] && { printf "cannot open '%s': Input/output error\n" "$last_arg" >&2; exit 1; } { exists "$last_arg" || [ -n "$(mark "$last_arg" 2)" ]; } && { echo "$last_arg"; exit 0; } printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1 fi ;; diff --git a/tests/page_dialog_test.cjs b/tests/page_dialog_test.cjs index e656b979..157d9054 100644 --- a/tests/page_dialog_test.cjs +++ b/tests/page_dialog_test.cjs @@ -48,7 +48,7 @@ window.$.post = (url, data, done) => { if (data.cmd === 'previewreplicate') { const answer = () => done(JSON.stringify(replicateAnswer(data))); if (holdReplicate) lateAnswers.push(answer); else answer(); } if (data.cmd === 'previewconvert') done(JSON.stringify(convertPreview)); if (data.cmd === 'previewschedules') done(JSON.stringify(schedulePreview)); - if (data.cmd === 'schedulesnapshots' && scheduleAnswer !== null) done(scheduleAnswer); + if (data.cmd === 'scheduledataset' && scheduleAnswer !== null) done(scheduleAnswer); const reply = window.$.Deferred(); replies.push(reply); return reply.promise(); @@ -445,7 +445,8 @@ async function confirm() { } // Schedules: starts at the dataset's own, shows the rows that fit its choice, and Save sends what is on screen and says the answer. { - schedulePreview = {every: 'days', time: '05:00', days: [1], keep: '1', inherited: 'the snapshot schedule of tank, every Tuesday at 06:00; it keeps all it takes'}; + schedulePreview = {every: 'days', time: '05:00', days: [1], keep: '1', inherited: 'the snapshot schedule of tank, every Tuesday at 06:00; it keeps all it takes', + copies: {every: 'hourly', time: '00:00', days: [0, 1, 2, 3, 4, 5, 6], keep: '3', under: 'usb', inherited: 'Replicate Every Dataset, under backup every hour at :30; it keeps all its snapshots', skipped: ''}}; const schedules = async () => { window.addDatasetContext('tank', 'tank/data', 0, 'menu', 0, 'none', '', 'filesystem'); menu.find(option => option.text === 'Schedules').action({preventDefault() {}}); @@ -473,9 +474,26 @@ async function confirm() { days().forEach(box => { box.checked = ['1', '3'].includes(box.value); }); field('time').value = '13:30'; field('keep').value = ' 2 '; + // Replication: its own copies, with Copy Under for a repeat and what Default follows. + const copy = id => window.document.getElementById('zfsm-copies-'+id); + const copyRows = () => ['days', 'time', 'keep', 'under'].map(row => visible('zfsm-copies-'+row+'-row')).concat(visible('zfsm-copies-default')); + const pickCopies = async value => { copy('every').value = value; copy('every').dispatchEvent(new window.Event('change')); await settle(); }; + assert.deepEqual([Array.from(copy('every').options, option => option.textContent), copy('every').value, copy('every').labels[0].textContent.startsWith('Replication:'), copyRows(), copy('under').value, copy('keep').value], + [['Default', 'None', 'Every hour', 'On days of the week'], 'hourly', true, [false, false, true, true, false], 'usb', '3'], 'its own copies are picked, with Copy Under and Keep'); + assert.deepEqual([rows(), copy('skipped')], [[true, true, true, false], null], 'Snapshots\' rows stay as they were, and one it copies says nothing of it'); + await pickCopies('default'); + assert.deepEqual([copyRows(), copy('default').textContent], [[false, false, false, false, true], 'Default follows Replicate Every Dataset, under backup every hour at :30; it keeps all its snapshots.'], 'Default says what it follows'); + await pickCopies('none'); + assert.deepEqual(copyRows(), [false, false, false, false, false], 'None has no rows'); + await pickCopies('days'); + window.document.querySelectorAll('.zfsm-copies-day').forEach(box => { box.checked = box.value === '6'; }); + copy('time').value = '02:15'; + copy('keep').value = ' 7d '; + copy('under').value = ' backup '; scheduleAnswer = JSON.stringify({said: 'Saved: tank/data', schedules_html: 'x'}); await confirm(); - assert.deepEqual(posts.at(-1), {cmd: 'schedulesnapshots', zdataset: 'tank/data', every: 'days', time: '13:30', days: '1,3', keep: '2'}, 'Save sends what is on screen'); + assert.deepEqual(posts.at(-1), {cmd: 'scheduledataset', zdataset: 'tank/data', every: 'days', time: '13:30', days: '1,3', keep: '2', copy_every: 'days', copy_time: '02:15', copy_days: '6', copy_keep: '7d', under: 'backup'}, + 'Save sends both parts as on screen'); const text = () => window.Swal2.getHtmlContainer().textContent; const result = () => [window.Swal2.getTitle().textContent, window.Swal2.getIcon().classList.contains('swal2-error'), text()]; assert.deepEqual(result(), ['Schedules Result', false, 'Saved: tank/data'], 'and shows what was saved, as text'); @@ -496,11 +514,16 @@ async function confirm() { await settle(); assert.deepEqual(result(), ['Schedules Result', true, 'The request failed. Reload the page and open Schedules to see what was saved.'], 'a request that fails says so'); schedulePreview = {every: 'default', time: '00:00', days: [0, 1, 2, 3, 4, 5, 6], keep: '', inherited: 'Snapshot Every Dataset, every hour at :30; it keeps all it takes', - skipped: 'tank/data has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool'}; + skipped: 'tank/data has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool', + copies: {every: 'default', time: '00:00', days: [0, 1, 2, 3, 4, 5, 6], keep: '', under: 'backup', inherited: 'Replicate Every Dataset, under backup every hour at :30; it keeps all its snapshots', + skipped: 'tank/data is in pool backup, where its copy would go'}}; await schedules(); const line = field('skipped'); assert.deepEqual([line?.textContent, line?.querySelectorAll('i').length, line?.parentElement.firstElementChild === line], ['tank/data has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool.', 0, true], 'one no run takes says why, first and as text'); + const unsent = window.document.getElementById('zfsm-copies-skipped'); + assert.deepEqual([unsent?.textContent, unsent?.querySelectorAll('i').length, unsent?.previousElementSibling.id], ['tank/data is in pool backup, where its copy would go.', 0, 'zfsm-copies-default'], + 'and one Replication leaves out says why under it, as text'); window.Swal2.close(); schedulePreview = {failed: {request: 'Invalid dataset name'}}; await schedules(); diff --git a/tests/phone_render_test.cjs b/tests/phone_render_test.cjs index 56dee25a..8d9bd5e8 100644 --- a/tests/phone_render_test.cjs +++ b/tests/phone_render_test.cjs @@ -54,9 +54,10 @@ let checked = 0; try { // The Replicate and Schedules dialogs with a repeat on days, as on the Main tab of a 390 px phone. const repeats = [['Replicate', "zfsmReplicateHtml({snapshots: ['tank/data@a']}, 'tank/data')", 'zfsm-replicate', 'zfsmReplicateRepeatSync'], - ['Schedules', "zfsmScheduleDialogHtml({every: 'days', time: '05:00', days: [1], keep: '', inherited: 'Snapshot Every Dataset, every hour at :30; it keeps all it takes', skipped: 'tank/appdata has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool'})", 'zfsm-schedules', 'zfsmScheduleDialogSync']]; + ['Schedules', "zfsmScheduleDialogHtml({every: 'days', time: '05:00', days: [1], keep: '', inherited: 'Snapshot Every Dataset, every hour at :30; it keeps all it takes', skipped: 'tank/appdata has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool', copies: {every: 'days', time: '06:00', days: [1], keep: '', under: 'backup', inherited: 'Replicate Every Dataset, which is off: no scheduled copies', skipped: ''}})", 'zfsm-schedules', 'zfsmScheduleDialogSync'], + ['Schedules', "zfsmScheduleDialogHtml({every: 'days', time: '05:00', days: [1], keep: '', inherited: 'Snapshot Every Dataset, every hour at :30; it keeps all it takes', skipped: 'tank/appdata has Sanoid or Syncoid snapshots (autosnap_, syncoid_), so it is left to that tool', copies: {every: 'days', time: '06:00', days: [1], keep: '', under: 'backup', inherited: 'Replicate Every Dataset, which is off: no scheduled copies', skipped: ''}})", 'zfsm-copies', 'zfsmScheduleDialogSync']]; for (const [dialog, html, prefix, sync] of repeats) for (const theme of ['black', 'white', 'azure', 'gray']) { - const where = `${dialog} dialog, ${theme} theme`; + const where = `${dialog} dialog (${prefix}), ${theme} theme`; const shown = render(where, 390, sheets(theme, 'main') + dialogs + link(path.join(frontend, 'ZFSMImpact.js')), ``, `() => { + const popup = Swal2.getPopup(), seen = id => { const p = popup.querySelector(id); return p.getBoundingClientRect().height > 0 && p.textContent.includes('${deep}'); }; + return {seen: [seen('#zfsm-schedules-default'), seen('#zfsm-copies-default')], wide: [Swal2.getHtmlContainer().scrollWidth, Swal2.getHtmlContainer().clientWidth, document.documentElement.scrollWidth, innerWidth]}; +}`); + assert.deepEqual(shown.seen, [true, true], `${where}: both Default lines must show what they follow: ${JSON.stringify(shown)}`); + assert.ok(shown.wide[0] <= shown.wide[1] && shown.wide[2] <= shown.wide[3], `${where}: nothing may scroll sideways in the dialog or past the screen: ${JSON.stringify(shown.wide)}`); + checked++; + } + // Snapshots Admin in Shadowbox's 342 px frame: rows of two datasets with long names, as the page prints them. const admin = page('ZFSMAdminDatasetSnaps.php'); const rows = ['tank/appdata/plex', 'tank/appdata/nextcloud'].flatMap(dataset => [1, 2, 3].map(n => `` diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 9e283d86..7949dcb5 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -194,14 +194,14 @@ function same($expected, $actual, $message) { $pool('datasets', array_merge($read('pool/datasets'), array("tank/enc\tfilesystem", "backup/tank/enc\tfilesystem"))); $pool('marks', array_merge($read('pool/marks'), array("tank/enc@a\t4242\t9000"))); $pool('props', array_merge($read('pool/props'), array("tank/enc\tencryption\taes-256-gcm", "backup/tank/enc\tzfs.master:holder\ton"))); -same(array('can_proceed' => false, 'error' => 'backup/tank/enc only holds the copies of datasets below this one, and zfs cannot receive the raw copy of this encrypted dataset over it: move aside (zfs rename) any copy below it that no schedule makes now, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all (a copy a repeat set in Replicate made asks for one Replicate by hand)'), +same(array('can_proceed' => false, 'error' => 'backup/tank/enc only holds the copies of datasets below this one, and zfs cannot receive the raw copy of this encrypted dataset over it: copy it elsewhere instead, or move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all; a copy destroyed loses the snapshots only it had (one a repeat set in Replicate made asks for one Replicate by hand)'), zfsmReplicatePlan('tank/enc', 'a', 'backup/tank/enc', array('estimate' => false)), 'a raw stream onto a holder'); // Nor any copy onto an encrypted holder, as one under an encrypted dataset is (live, 2.4.4); one not read is not sent onto. $pool('datasets', array_merge($read('pool/datasets'), array("tank/sealed\tfilesystem", "backup/tank/sealed\tfilesystem"))); $pool('marks', array_merge($read('pool/marks'), array("tank/sealed@a\t4646\t9004"))); $pool('props', array_merge($read('pool/props'), array("backup/tank/sealed\tzfs.master:holder\ton", "backup/tank/sealed\tencryption\taes-256-gcm"))); $plan = function() { return zfsmReplicatePlan('tank/sealed', 'a', 'backup/tank/sealed', array('estimate' => false)); }; -same(array('can_proceed' => false, 'error' => 'backup/tank/sealed only holds the copies of datasets below this one, and zfs cannot receive a copy over it, as it is encrypted: move aside (zfs rename) any copy below it that no schedule makes now, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all (a copy a repeat set in Replicate made asks for one Replicate by hand)'), +same(array('can_proceed' => false, 'error' => 'backup/tank/sealed only holds the copies of datasets below this one, and zfs cannot receive a copy over it, as it is encrypted: copy it elsewhere instead, or move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all; a copy destroyed loses the snapshots only it had (one a repeat set in Replicate made asks for one Replicate by hand)'), $plan(), 'a stream onto an encrypted holder'); putenv('ZFSM_TEST_CRYPT_FAIL=backup/tank/sealed'); same(array('can_proceed' => false, 'error' => 'backup/tank/sealed only holds the copies of datasets below this one, and whether it is encrypted could not be read'), $plan(), 'one whose encryption could not be read'); diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index dd3d1252..34b960f6 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -144,6 +144,33 @@ function same($expected, $actual, $message) { $save('tank/data/sub', $part('default'), $part('default')); $save('tank/docker', $part('default'), $part('default')); same(array('', '', ''), $nearer, 'copies below snapshots set to None there, and snapshots of one with its own copies, clash with neither above'); +// A part set to Default hands its datasets to the one above: one as saved is checked against that, in its minute or names, but a +// clash there before stays the owner's. Either part. +$apps = $read('pool/datasets'); +$pool('datasets', array_merge($apps, array("tank/apps\tfilesystem", "tank/apps/db\tfilesystem", "tank/apps/log\tfilesystem", "tank/mv\tfilesystem"))); +$ymd = function($zdataset, $snapshots, $copies) use ($cfg) { return zfsmScheduleDialogSave($zdataset, $snapshots, $copies, array('snap_pattern' => 'Ymd') + $cfg); }; +$handed = array($save('tank/apps', $part('days', '03:00', '1'), $part('none'))['saved'], $save('tank/apps/db', $part('days', '05:00', '1'), $part('days', '04:00', '1', 0, 'usb'))['saved'], + $save('tank/apps/log', $part('days', '06:00', '1'), $part('default'))['saved']); +$handed[] = $ymd('tank/apps/log', $part('default'), $part('default'))['saved']; +$handed[] = preg_match('/^Nothing was saved: for (the copies of tank\/apps\/db and below|the snapshot schedule of tank\/apps), Snapshots Pattern would give the run at .* the same snapshot name as .*; add the time to Snapshots Pattern$/', + $ymd('tank/apps/db', $part('default'), $part('days', '04:00', '1', 0, 'usb'))['why']); +$handed[] = $save('tank/apps/db', $part('default'), $part('days', '04:00', '1', 0, 'usb'))['saved']; +$handed[] = $save('tank/apps', $part('days', '03:00', '1'), $part('days', '07:00', '1', 0, 'usb'))['saved']; +$handed[] = $save('tank/apps/db', $part('days', '07:00', '1'), $part('days', '04:00', '1', 0, 'usb'))['saved']; +$handed[] = preg_match('/^Nothing was saved: for (the copies of tank\/apps and below|the snapshot schedule of tank\/apps\/db), the .* starts then too; pick another time$/', + $save('tank/apps/db', $part('days', '07:00', '1'), $part('default'))['why']); +same(array(true, true, true, true, 1, true, true, true, 1), $handed, 'Default is checked against the one it hands its datasets to, as saved, but a clash there before is left'); +// Copies of a dataset on the day the snapshots of another run: renamed below that one, the rename says the clash it makes, by its +// Snapshots Pattern. +same(true, $save('tank/mv', $part('default'), $part('days', '04:00', '1', 0, 'usb'))['saved'], 'copies of one apart'); +$moved = renameDataset('tank/mv', 'tank/apps/mv', 0, array('snap_pattern' => 'Ymd') + $cfg); +same(array(array('tank/apps/mv' => 0), array('Schedules of tank/apps/mv'), 1, array('tank/apps/mv')), array($moved['succeeded'], array_keys($moved['failed']), + preg_match('/^moved with it, but for (the copies of tank\/apps\/mv and below|the snapshot schedule of tank\/apps), Snapshots Pattern would give the run at .* the same snapshot name as .*; add the time to Snapshots Pattern$/', + $moved['failed']['Schedules of tank/apps/mv'] ?? ''), + array_values(preg_grep('~^tank/(apps/)?mv$~', array_column($stored(), 'dataset')))), 'a rename into a clash says it, and its schedule moves'); +same(array(true, true, true), array_map(function($zdataset) use ($save, $part) { return $save($zdataset, $part('default'), $part('default'))['saved']; }, array('tank/apps/db', 'tank/apps/mv', 'tank/apps')), + 'back to Default'); +$pool('datasets', $apps); // None: no copies of it and below; Copy Under starts at what Default would follow. same('Saved: tank/data follows Snapshot Every Dataset, which is off: no scheduled snapshots. Saved: no scheduled copies of tank/data and below, but for datasets with their own and repeats set in Replicate; the copies made stay', @@ -182,20 +209,28 @@ function same($expected, $actual, $message) { $t = 1800000000; $sends = function($pattern) use ($read) { return count(preg_grep($pattern, $read('calls.log'))); }; same(0, zfsmScheduleRun($id, $cfg, $t), 'the first copies'); -same(array('succeeded' => array('tank/new' => 0), 'failed' => array()), renameDataset('tank/data', 'tank/new', 0), 'a rename'); +putenv('ZFSM_TEST_PHP='.PHP_BINARY); +putenv('ZFSM_TEST_LOCK_PROBE='.zfsmJobLockFile('tank/data', 'replicate')); +same(array('succeeded' => array('tank/new' => 0), 'failed' => array()), renameDataset('tank/data', 'tank/new', 0, $cfg), 'a rename'); +putenv('ZFSM_TEST_LOCK_PROBE'); +same(array('held', 'held'), $read('pool/probe.log'), 'its replication lock is held while it and its copy are renamed'); same(array('backup/tank/new', 'backup/tank/new/sub'), array_values(preg_grep('~^backup/tank/(data|new)~', $datasets())), 'its copy and the one below follow it'); $bookmarks = function($under) use ($read) { return array_values(preg_grep('~^'.preg_quote($under, '~').'[/#]~', preg_grep('/#/', array_map(function($line) { return explode("\t", $line)[0]; }, $read('pool/marks'))))); }; same(array('tank/new#'.zfsmReplicaBookmarkPrefix('backup/tank/new').'auto-20270115080000', 'tank/new/sub#'.zfsmReplicaBookmarkPrefix('backup/tank/new/sub').'auto-20270115080000'), $bookmarks('tank/new'), 'their bookmarks take the copies\' new names'); $reset(); +// Its snapshots gone from the source, only the bookmarks that took the new names keep the next run incremental. +$pool('marks', array_values(preg_grep('~^tank/new(/sub)?@auto-20270115080000\t~', $read('pool/marks'), PREG_GREP_INVERT))); same(0, zfsmScheduleRun($id, $cfg, $t + 3600), 'the next run'); -same(array(1, 0), array($sends('~^send -P -v -L -i \S+ tank/new@~'), $sends('~^send -P -v -L tank/new@~')), 'sends only the changes to the renamed copy'); +$from = function($zdataset) { return preg_quote($zdataset.'#'.zfsmReplicaBookmarkPrefix('backup/'.$zdataset).'auto-20270115080000 '.$zdataset.'@', '~'); }; +same(array(1, 1, 0), array($sends('~^send -P -v -L -i '.$from('tank/new').'~'), $sends('~^send -P -v -L -i '.$from('tank/new/sub').'~'), $sends('~^send -P -v -L tank/new@~')), + 'sends from the moved bookmarks only the changes to the renamed copies'); // A run cut off part-way, then a rename: zfs finds the snapshot by guid under its new name (live, 2.4.4), so the next run carries it on. putenv('ZFSM_TEST_SEND_CUT=1'); zfsmScheduleRun($id, $cfg, $t + 7200); putenv('ZFSM_TEST_SEND_CUT'); $tokens = function($copy) use ($read) { return array_values(preg_grep('~^'.preg_quote($copy, '~').'(/sub)?\treceive_resume_token\ttok:~', $read('pool/props'))); }; -same(array(2, array('succeeded' => array('tank/cut' => 0), 'failed' => array())), array(count($tokens('backup/tank/new')), renameDataset('tank/new', 'tank/cut', 0)), +same(array(2, array('succeeded' => array('tank/cut' => 0), 'failed' => array())), array(count($tokens('backup/tank/new')), renameDataset('tank/new', 'tank/cut', 0, $cfg)), 'a rename after a run cut off part-way'); $reset(); $marks = function() use ($read) { return array_map(function($line) { return explode("\t", $line)[0]; }, $read('pool/marks')); }; @@ -204,72 +239,138 @@ function same($expected, $actual, $message) { // Into a dataset whose copy is not made yet: an empty holder takes its place first, as zfs rename needs the parent. $pool('datasets', array_merge($read('pool/datasets'), array("tank/deep\tfilesystem"))); $reset(); -same(array('succeeded' => array('tank/deep/data' => 0), 'failed' => array()), renameDataset('tank/cut', 'tank/deep/data', 0), 'a rename into a new dataset'); +same(array('succeeded' => array('tank/deep/data' => 0), 'failed' => array()), renameDataset('tank/cut', 'tank/deep/data', 0, $cfg), 'a rename into a new dataset'); same(array(array('create -o canmount=off -o readonly=on -o zfs.master:holder=on backup/tank/deep'), true, true), array(array_values(preg_grep('/^create /', $read('calls.log'))), zfsmReplicaHolder(array('server' => ''), 'backup/tank/deep'), in_array('backup/tank/deep/data/sub', $datasets(), true)), 'a holder, then the copy'); +// Into an encrypted one whose first copy is due: a holder there would stop that copy, so the copy stays, and why. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/vault\tfilesystem", "tank/vault/data\tfilesystem", "tank/vault/in\tfilesystem", "backup/tank/vault\tfilesystem", "backup/tank/vault/data\tfilesystem"))); +$pool('props', array_merge($read('pool/props'), array_map(function($zdataset) { return $zdataset."\tencryption\taes-256-gcm"; }, array('tank/vault', 'tank/vault/data', 'tank/vault/in')))); +$pool('marks', array_merge($read('pool/marks'), array("tank/vault/data@s\t97\t502", "backup/tank/vault/data@s\t97\t502"))); +same(array('Copy backup/tank/vault/data' => 'not renamed with it: zfs could not receive the copy due at backup/tank/vault/in over the dataset made there to hold this one, as one of the two is encrypted. It stays under its old name, so the next copy of the renamed dataset is a full one'), + renameDataset('tank/vault/data', 'tank/vault/in/data', 0, $cfg)['failed'], 'a rename into an encrypted dataset whose first copy is due'); +same(array(false, true), array(in_array('backup/tank/vault/in', $datasets(), true), in_array('backup/tank/vault/data', $datasets(), true)), 'no holder made, and the copy stays'); +// Whether either is encrypted not read: the same, and why. +renameDataset('tank/vault/in/data', 'tank/vault/data', 0, $cfg); +putenv('ZFSM_TEST_CRYPT_FAIL=tank/vault/in'); +same(array('Copy backup/tank/vault/data' => 'not renamed with it: whether a dataset made at backup/tank/vault/in to hold this one would stop the copy due there could not be read. It stays under its old name, so the next copy of the renamed dataset is a full one'), + renameDataset('tank/vault/data', 'tank/vault/in/data', 0, $cfg)['failed'], 'one whose encryption could not be read'); +putenv('ZFSM_TEST_CRYPT_FAIL'); +same(array(false, true), array(in_array('backup/tank/vault/in', $datasets(), true), in_array('backup/tank/vault/data', $datasets(), true)), 'no holder made there either'); // When it can't, the rename still stands and each copy left behind says why, by its name. $copy = function($zdataset) { return 'Copy backup/'.$zdataset; }; putenv('ZFSM_TEST_RENAME_FAIL=backup/tank/deep/data'); same(array('succeeded' => array('tank/moved' => 0), 'failed' => array($copy('tank/deep/data') => 'not renamed with it: cannot rename \'backup/tank/deep/data\': dataset is busy. It stays under its old name, so the next copy of the renamed dataset is a full one')), - renameDataset('tank/deep/data', 'tank/moved', 0), 'a copy that is busy'); + renameDataset('tank/deep/data', 'tank/moved', 0, $cfg), 'a copy that is busy'); putenv('ZFSM_TEST_RENAME_FAIL'); $add = function($names) use ($pool, $read) { $pool('datasets', array_merge($read('pool/datasets'), array_map(function($name) { return $name."\tfilesystem"; }, $names))); }; $add(array('backup/tank/again')); -same(array($copy('tank/docker') => 'left where it is, as a dataset already has its new name'), renameDataset('tank/docker', 'tank/again', 0)['failed'], 'a name already taken there'); -same(array('succeeded' => array('tank/again' => 0), 'failed' => array()), renameDataset('tank/again', 'tank/again', 0), 'a rename to its own name, which zfs does nothing for'); -// Its copy in a pool not imported now: zfs says the copy does not exist, so it is told as left behind. +same(array($copy('tank/docker') => 'left where it is, as a dataset already has its new name'), renameDataset('tank/docker', 'tank/again', 0, $cfg)['failed'], 'a name already taken there'); +same(array('succeeded' => array('tank/again' => 0), 'failed' => array()), renameDataset('tank/again', 'tank/again', 0, $cfg), 'a rename to its own name, which zfs does nothing for'); +// Its copy in a pool not imported now: zfs says the copy does not exist, so it is told as not checked; where no schedule copies it, +// nothing is said. +$add(array('tank/o')); +same(true, $save('tank/o', $part('default'), $part('none'))['saved'], 'None'); $all = $read('pool/datasets'); $pool('datasets', array_values(preg_grep('~^backup[\t/]~', $all, PREG_GREP_INVERT))); -$gone = renameDataset('tank/again', 'tank/away', 0)['failed']; +$gone = array(renameDataset('tank/again', 'tank/away', 0, $cfg)['failed'], renameDataset('tank/o', 'tank/o2', 0, $cfg)['failed']); $pool('datasets', array_merge($read('pool/datasets'), preg_grep('~^backup[\t/]~', $all))); -same(array($copy('tank/again') => 'not renamed with it, as its pool is not imported now. It stays under its old name, so the next copy of the renamed dataset is a full one'), $gone, - 'a copy in a pool not imported is told'); +same(array(array($copy('tank/again') => 'not checked, as its pool is not imported now: if it is there, it keeps its old name, and the next copy of the renamed dataset is a full one'), array()), + $gone, 'a copy in a pool not imported is told as not checked, and one nothing copies there not at all'); // A copy of the renamed dataset: its newest snapshot is on it. $copied = function($zdataset, $guid) use ($pool, $read) { $pool('marks', array_merge($read('pool/marks'), array("$zdataset@s\t$guid\t500", "backup/$zdataset@s\t$guid\t500"))); }; +// Under a copy that is encrypted, as in an encrypted pool, a holder is too; and a repeat set in Replicate due where no schedule +// copies. Each stops the copy due there, so the copy stays. +$add(array('tank/sealed', 'tank/sealed/data', 'tank/sealed/in', 'backup/tank/sealed', 'backup/tank/sealed/data', 'tank/vault/by', 'tank/vault/x', 'backup/tank/vault/x')); +$copied('tank/sealed/data', 71); +$copied('tank/vault/x', 72); +$pool('props', array_merge($read('pool/props'), array("backup/tank/sealed\tencryption\taes-256-gcm", "backup/tank/sealed/data\tencryption\taes-256-gcm", "tank/vault/by\tencryption\taes-256-gcm", + "tank/vault/x\tencryption\taes-256-gcm"))); +same(array(true, true), array($save('tank/vault/by', $part('default'), $part('none'))['saved'], $save('tank/vault/x', $part('default'), $part('days', '05:00', '1', 0, 'backup'))['saved']), 'None, and own copies'); +same(true, zfsmScheduleChange(function($jobs) { return $jobs + array('5e7a1b2c' => array('id' => '5e7a1b2c', 'dataset' => 'tank/vault/by', 'dest' => 'backup/tank/vault/by', 'server' => '', 'every' => 'daily', 'hour' => 3, 'day' => 0)); })['saved'], + 'a repeat set in Replicate'); +$blocked = function($level) { return 'not renamed with it: zfs could not receive the copy due at '.$level.' over the dataset made there to hold this one, as one of the two is encrypted. It stays under its old name, so the next copy of the renamed dataset is a full one'; }; +same(array(array($copy('tank/sealed/data') => $blocked('backup/tank/sealed/in')), array($copy('tank/vault/x') => $blocked('backup/tank/vault/by')), array(false, false, true, true)), + array(renameDataset('tank/sealed/data', 'tank/sealed/in/data', 0, $cfg)['failed'], renameDataset('tank/vault/x', 'tank/vault/by/x', 0, $cfg)['failed'], + array_map(function($zdataset) use ($datasets) { return in_array($zdataset, $datasets(), true); }, array('backup/tank/sealed/in', 'backup/tank/vault/by', 'backup/tank/sealed/data', 'backup/tank/vault/x'))), + 'under an encrypted copy, and where a repeat set in Replicate is due: no holder made, and each copy stays'); $add(array('tank/r', 'backup/tank/r', 'tank/q', 'backup/tank/q')); $copied('tank/q', 81); putenv('ZFSM_TEST_LIST_FAIL=backup/tank/r'); -same(array($copy('tank/r') => 'not renamed with it, as whether it is there could not be read'), renameDataset('tank/r', 'tank/r2', 0)['failed'], 'a copy that could not be read'); +same(array($copy('tank/r') => 'not renamed with it, as whether it is there could not be read'), renameDataset('tank/r', 'tank/r2', 0, $cfg)['failed'], 'a copy that could not be read'); putenv('ZFSM_TEST_LIST_FAIL=backup/tank/q2'); -same(array($copy('tank/q') => 'not renamed with it, as whether its new name is free could not be read'), renameDataset('tank/q', 'tank/q2', 0)['failed'], 'a new name that could not be read'); +same(array($copy('tank/q') => 'not renamed with it, as whether its new name is free could not be read'), renameDataset('tank/q', 'tank/q2', 0, $cfg)['failed'], 'a new name that could not be read'); putenv('ZFSM_TEST_LIST_FAIL'); $add(array('tank/up', 'tank/u', 'backup/tank/u')); $copied('tank/u', 82); putenv('ZFSM_TEST_LIST_FAIL=backup/tank/up'); same(array($copy('tank/u') => 'not renamed with it: the datasets above its new name could not be read. It stays under its old name, so the next copy of the renamed dataset is a full one'), - renameDataset('tank/u', 'tank/up/u', 0)['failed'], 'one whose new parent could not be read'); + renameDataset('tank/u', 'tank/up/u', 0, $cfg)['failed'], 'one whose new parent could not be read'); putenv('ZFSM_TEST_LIST_FAIL'); +// A new name that leaves the copy of it or one below it no room for the bookmark a copy keeps: nothing is renamed. One with no room +// already may still be renamed. +$long = 'tank/'.str_repeat('l', 80); +$kid = str_repeat('m', 64); +$add(array('tank/k', 'tank/k/'.$kid, $long)); +same(array(array('succeeded' => array(), 'failed' => array('tank/kk' => 'Nothing was renamed: the names of tank/kk/'.$kid.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name')), true), + array(renameDataset('tank/k', 'tank/kk', 0, $cfg), in_array('tank/k/'.$kid, $datasets(), true)), 'a new name that leaves no room for a copy\'s bookmark'); +same(array($long.'2' => 0), renameDataset($long, $long.'2', 0, $cfg)['succeeded'], 'one with no room already'); +// Its own copies go with it, so the room is that under their Copy Under. +$far = 'backup/'.str_repeat('c', 20); +$kin = str_repeat('n', 50); +$add(array($far, 'tank/j', 'tank/j/'.$kin)); +same(true, $save('tank/j', $part('default'), $part('days', '05:00', '1', 0, $far))['saved'], 'own copies under a long name'); +same(array(array('tank/jj' => 'Nothing was renamed: the names of tank/jj/'.$kin.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'), true), + array(renameDataset('tank/j', 'tank/jj', 0, $cfg)['failed'], $save('tank/j', $part('default'), $part('default'))['saved']), 'the room under its own Copy Under'); +// A replication of it or one below it running: nothing is renamed, and the locks it took are let go. +$add(array('tank/lk', 'tank/lk/kid')); +$lock = zfsmJobLock(zfsmJobLockFile('tank/lk/kid', 'replicate'), false); +$busy = renameDataset('tank/lk', 'tank/lk2', 0, $cfg); +zfsmJobUnlock($lock); +$loose = zfsmJobLock(zfsmJobLockFile('tank/lk', 'replicate'), false); +same(array(array('succeeded' => array(), 'failed' => array('tank/lk2' => 'Nothing was renamed: a replication of tank/lk/kid is running')), true, true), + array($busy, is_resource($loose), in_array('tank/lk/kid', $datasets(), true)), 'nothing renamed while one below it copies, and its lock let go'); +zfsmJobUnlock($loose); +putenv('ZFSM_TEST_TREE_FAIL=1'); +$unread = renameDataset('tank/lk', 'tank/lk2', 0, $cfg)['failed']; +putenv('ZFSM_TEST_TREE_FAIL'); +file_put_contents($work.'/plain', ''); +putenv('ZFSM_TEST_JOB_DIR='.$work.'/plain'); +$unopened = renameDataset('tank/lk', 'tank/lk2', 0, $cfg)['failed']; +putenv('ZFSM_TEST_JOB_DIR='.$work.'/state'); +same(array(array('tank/lk2' => 'Nothing was renamed: the datasets at and below it could not be read: cannot open \'tank/lk\': Input/output error'), + array('tank/lk2' => 'Nothing was renamed: the replication lock of tank/lk could not be opened'), true), array($unread, $unopened, in_array('tank/lk/kid', $datasets(), true)), + 'nor when they could not be read or locked'); // One whose own Replication is None: nothing copies it there, so its copy stays. $add(array('tank/n', 'backup/tank/n')); same(true, $save('tank/n', $part('default'), $part('none'))['saved'], 'None'); -same(array($copy('tank/n') => 'left where it is, as no schedule copies the renamed dataset or one below it there'), renameDataset('tank/n', 'tank/final', 0)['failed'], 'nothing copies it there'); +same(array($copy('tank/n') => 'left where it is, as no schedule copies the renamed dataset or one below it there'), renameDataset('tank/n', 'tank/final', 0, $cfg)['failed'], 'nothing copies it there'); same(array(true, false), array(in_array('backup/tank/n', $datasets(), true), in_array('backup/tank/final', $datasets(), true)), 'it stays'); // One below it still copied there: the holder over that copy follows the rename too. $add(array('tank/p', 'tank/p/kid', 'backup/tank/p', 'backup/tank/p/kid')); $pool('props', array_merge($read('pool/props'), array("backup/tank/p\tzfs.master:holder\ton"))); same(array(true, true), array($save('tank/p', $part('default'), $part('none'))['saved'], $save('tank/p/kid', $part('default'), $part('hourly', '', '', 0, 'backup'))['saved']), 'None above, its own below'); -same(array('succeeded' => array('tank/p2' => 0), 'failed' => array()), renameDataset('tank/p', 'tank/p2', 0), 'a rename of the one above'); +same(array('succeeded' => array('tank/p2' => 0), 'failed' => array()), renameDataset('tank/p', 'tank/p2', 0, $cfg), 'a rename of the one above'); same(array('backup/tank/p2', 'backup/tank/p2/kid'), array_values(preg_grep('~^backup/tank/p~', $datasets())), 'its copy and the one below follow it'); // Its schedules not moved, as another change holds them: its copies follow as they would have, ready for its schedules set again. $lock = zfsmJobLock(zfsmJobDir('schedule').'/settings.lock', false); -same(array('Schedules of tank/p2'), array_keys(renameDataset('tank/p2', 'tank/p3', 0)['failed']), 'only the schedules are said'); +same(array('Schedules of tank/p2'), array_keys(renameDataset('tank/p2', 'tank/p3', 0, $cfg)['failed']), 'only the schedules are said'); zfsmJobUnlock($lock); same(array('backup/tank/p3', 'backup/tank/p3/kid'), array_values(preg_grep('~^backup/tank/p~', $datasets())), 'its copies follow it'); // One with no copy there: nothing to say. -same(array('succeeded' => array('tank/r3' => 0), 'failed' => array()), renameDataset('tank/r2', 'tank/r3', 0), 'one with no copy'); +same(array('succeeded' => array('tank/r3' => 0), 'failed' => array()), renameDataset('tank/r2', 'tank/r3', 0, $cfg), 'one with no copy'); // Not a copy of the renamed dataset, as one left by a destroy before a dataset of that name was made again: it stays, and why. $add(array('tank/a', 'backup/tank/a', 'tank/e', 'backup/tank/e', 'tank/f', 'backup/tank/f', 'tank/g', 'backup/tank/g')); $pool('marks', array_merge($read('pool/marks'), array("tank/a@new\t96\t501", "backup/tank/a@old\t95\t400"))); -same(array($copy('tank/a') => 'left where it is, as its newest snapshot is not on the renamed dataset'), renameDataset('tank/a', 'tank/a2', 0)['failed'], 'a copy of another dataset'); -same(array($copy('tank/e') => 'left where it is, as it holds no snapshot of the renamed dataset'), renameDataset('tank/e', 'tank/e2', 0)['failed'], 'one with no snapshot that is no holder'); +same(array($copy('tank/a') => 'left where it is, as its newest snapshot is not on the renamed dataset'), renameDataset('tank/a', 'tank/a2', 0, $cfg)['failed'], 'a copy of another dataset'); +same(array($copy('tank/e') => 'left where it is, as it holds no snapshot of the renamed dataset'), renameDataset('tank/e', 'tank/e2', 0, $cfg)['failed'], 'one with no snapshot that is no holder'); $copied('tank/f', 83); $copied('tank/g', 87); putenv('ZFSM_TEST_MARKS_FAIL=backup/tank/f'); -same(array($copy('tank/f') => 'not renamed with it, as its snapshots could not be read'), renameDataset('tank/f', 'tank/f2', 0)['failed'], 'one whose snapshots could not be read'); +same(array($copy('tank/f') => 'not renamed with it, as its snapshots could not be read'), renameDataset('tank/f', 'tank/f2', 0, $cfg)['failed'], 'one whose snapshots could not be read'); putenv('ZFSM_TEST_MARKS_FAIL=tank/g2'); -same(array($copy('tank/g') => 'not renamed with it, as the snapshots of the renamed dataset could not be read'), renameDataset('tank/g', 'tank/g2', 0)['failed'], 'nor those of the renamed one'); +same(array($copy('tank/g') => 'not renamed with it, as the snapshots of the renamed dataset could not be read'), renameDataset('tank/g', 'tank/g2', 0, $cfg)['failed'], 'nor those of the renamed one'); putenv('ZFSM_TEST_MARKS_FAIL'); same(array('backup/tank/a', 'backup/tank/e', 'backup/tank/f', 'backup/tank/g'), array_values(preg_grep('~^backup/tank/[aefg]2?$~', $datasets())), 'they stay'); @@ -282,22 +383,22 @@ function same($expected, $actual, $message) { $marked('tank/b1', 84); putenv('ZFSM_TEST_BOOKMARK_FAIL=1'); same(array($copy('tank/b1') => 'renamed with it, but not all its bookmarks were: cannot create bookmark \'tank/b1x#'.zfsmReplicaBookmarkPrefix('backup/tank/b1x').'s\': permission denied'), - renameDataset('tank/b1', 'tank/b1x', 0)['failed'], 'a bookmark not made'); + renameDataset('tank/b1', 'tank/b1x', 0, $cfg)['failed'], 'a bookmark not made'); putenv('ZFSM_TEST_BOOKMARK_FAIL'); $marked('tank/b2', 85); putenv('ZFSM_TEST_BOOKMARKS_FAIL=1'); -same(array($copy('tank/b2') => 'renamed with it, but its bookmarks could not be read: cannot open \'tank/b2x\': Input/output error'), renameDataset('tank/b2', 'tank/b2x', 0)['failed'], 'its bookmarks not read'); +same(array($copy('tank/b2') => 'renamed with it, but its bookmarks could not be read: cannot open \'tank/b2x\': Input/output error'), renameDataset('tank/b2', 'tank/b2x', 0, $cfg)['failed'], 'its bookmarks not read'); putenv('ZFSM_TEST_BOOKMARKS_FAIL'); $marked('tank/b3', 86); putenv('ZFSM_TEST_DESTROY_FAIL=tank/b3x#'.zfsmReplicaBookmarkPrefix('backup/tank/b3').'s'); same(array($copy('tank/b3') => 'renamed with it, but not all its bookmarks were: cannot destroy \'tank/b3x#'.zfsmReplicaBookmarkPrefix('backup/tank/b3').'s\': Input/output error'), - renameDataset('tank/b3', 'tank/b3x', 0)['failed'], 'the old one not removed'); + renameDataset('tank/b3', 'tank/b3x', 0, $cfg)['failed'], 'the old one not removed'); putenv('ZFSM_TEST_DESTROY_FAIL'); same(array('backup/tank/b1x', 'backup/tank/b2x', 'backup/tank/b3x'), array_values(preg_grep('~^backup/tank/b\d~', $datasets())), 'each copy renamed'); // A bookmark name can end in a space (OpenZFS), the last one listed too. $marked('tank/b4', 91); $pool('marks', preg_replace('~^(tank/b4#[^\t]*s)\t~', "$1 \t", $read('pool/marks'))); -same(array(array(), array('tank/b4x#'.zfsmReplicaBookmarkPrefix('backup/tank/b4x').'s ')), array(renameDataset('tank/b4', 'tank/b4x', 0)['failed'], $bookmarks('tank/b4x')), +same(array(array(), array('tank/b4x#'.zfsmReplicaBookmarkPrefix('backup/tank/b4x').'s ')), array(renameDataset('tank/b4', 'tank/b4x', 0, $cfg)['failed'], $bookmarks('tank/b4x')), 'a bookmark whose name ends in a space takes the copy\'s new name'); // A stopped run's holds take the copy's new name, so the run that carries it on takes them off; another hold stays. $tag = function($zdest) { return zfsmKeepHoldTag(zfsmKeepTag('', $zdest)); }; @@ -308,7 +409,7 @@ function same($expected, $actual, $message) { $holds = function($zdataset) use ($read) { return array_values(preg_grep('~^'.preg_quote($zdataset, '~').'@~', $read('pool/holds'))); }; $held('tank/h1', 88); $held('tank/h1/k', 89); -same(array('succeeded' => array('tank/h1x' => 0), 'failed' => array()), renameDataset('tank/h1', 'tank/h1x', 0), 'a copy a stopped run holds'); +same(array('succeeded' => array('tank/h1x' => 0), 'failed' => array()), renameDataset('tank/h1', 'tank/h1x', 0, $cfg), 'a copy a stopped run holds'); same(array(array("tank/h1x@s\tkeep", "tank/h1x@s\t".$tag('backup/tank/h1x')), array("tank/h1x/k@s\tkeep", "tank/h1x/k@s\t".$tag('backup/tank/h1x/k'))), array($holds('tank/h1x'), $holds('tank/h1x/k')), 'its hold and the one below take the new names'); foreach (array('h2' => array('ZFSM_TEST_HELD_FAIL', 'its holds could not be read: cannot open \'tank/h2x\': Input/output error'), @@ -317,14 +418,14 @@ function same($expected, $actual, $message) { 'h5' => array('ZFSM_TEST_RELEASE_FAIL', 'not all its holds were: tank/h5x@s (cannot release hold from snapshot \'tank/h5x@s\': permission denied)')) as $name => list($knob, $why)) { $held('tank/'.$name, 90 + (int)substr($name, 1)); putenv($knob.'=1'); - same(array($copy('tank/'.$name) => 'renamed with it, but '.$why), renameDataset('tank/'.$name, 'tank/'.$name.'x', 0)['failed'], $knob.': said'); + same(array($copy('tank/'.$name) => 'renamed with it, but '.$why), renameDataset('tank/'.$name, 'tank/'.$name.'x', 0, $cfg)['failed'], $knob.': said'); putenv($knob); } same(array("tank/h4x@s\t".$tag('backup/tank/h4'), "tank/h4x@s\tkeep"), $holds('tank/h4x'), 'a hold not moved stays as it was'); // A snapshot name can end in a space (OpenZFS). $held('tank/h6', 96); foreach (array('marks', 'holds') as $file) $pool($file, preg_replace('~^tank/h6@s\t~', "tank/h6@s \t", $read('pool/'.$file))); -same(array(array('tank/h6x' => 0), array("tank/h6x@s \tkeep", "tank/h6x@s \t".$tag('backup/tank/h6x'))), array(renameDataset('tank/h6', 'tank/h6x', 0)['succeeded'], $holds('tank/h6x')), +same(array(array('tank/h6x' => 0), array("tank/h6x@s \tkeep", "tank/h6x@s \t".$tag('backup/tank/h6x'))), array(renameDataset('tank/h6', 'tank/h6x', 0, $cfg)['succeeded'], $holds('tank/h6x')), 'the hold on a snapshot whose name ends in a space takes the new name'); same(array('saved' => false, 'why' => 'Nothing was saved: that is no dataset name', 'said' => ''), $save('', $part('hourly'), $part('hourly', '', '', 0, 'usb')), 'the dialog saves no Settings schedule'); @@ -333,7 +434,7 @@ function same($expected, $actual, $message) { $copied('tank/w', 88); $text = file_get_contents(zfsmScheduleFile()); file_put_contents(zfsmScheduleFile(), '{'); -$answer = renameDataset('tank/w', 'tank/w2', 0); +$answer = renameDataset('tank/w', 'tank/w2', 0, $cfg); file_put_contents(zfsmScheduleFile(), $text); same(array(array('Schedules of tank/w', 'Copies of tank/w'), 'not renamed with it, as the schedules that say where they go could not be read', array('backup/tank/w')), array(array_keys($answer['failed']), $answer['failed']['Copies of tank/w'] ?? '', array_values(preg_grep('~^backup/tank/w~', $datasets()))), 'schedules not read: copies not renamed, and said'); diff --git a/tests/schedule_snapshot_test.php b/tests/schedule_snapshot_test.php index 2bee76b1..3620d0e1 100644 --- a/tests/schedule_snapshot_test.php +++ b/tests/schedule_snapshot_test.php @@ -354,14 +354,14 @@ function same($expected, $actual, $message) { touch(zfsmScheduleStateFile('5b5b5b5b')); $ids = array_column($stored(), 'id', 'dataset'); $own = function() use ($stored) { $by = array_column($stored(), 'id', 'dataset'); ksort($by); return $by; }; -same(array('succeeded' => array('tank/new' => 0), 'failed' => array()), renameDataset('tank/data', 'tank/new', 0), 'a rename'); +same(array('succeeded' => array('tank/new' => 0), 'failed' => array()), renameDataset('tank/data', 'tank/new', 0, $cfg), 'a rename'); same(array(array('tank/docker' => $ids['tank/docker'], 'tank/new' => $ids['tank/data'], 'tank/new/sub' => $ids['tank/data/sub']), 3, false, false), array($own(), count($stored()), is_file(zfsmScheduleStateFile('5a5a5a5a')), is_file(zfsmScheduleStateFile('5b5b5b5b'))), 'its own schedules and those below follow it'); $inode = fileinode(zfsmScheduleFile()); -same(array(array('succeeded' => array('tank/vm2' => 0), 'failed' => array()), $inode), array(renameDataset('tank/vm', 'tank/vm2', 0), fileinode(zfsmScheduleFile())), 'one with none writes nothing'); +same(array(array('succeeded' => array('tank/vm2' => 0), 'failed' => array()), $inode), array(renameDataset('tank/vm', 'tank/vm2', 0, $cfg), fileinode(zfsmScheduleFile())), 'one with none writes nothing'); $lock = zfsmJobLock(zfsmJobDir('schedule').'/settings.lock', false); same(array('succeeded' => array('tank/newer' => 0), 'failed' => array('Schedules of tank/new' => 'not moved to tank/newer (Another change to the schedules is being saved; try again): Remove them in Settings and set them again under Actions, Schedules')), - renameDataset('tank/new', 'tank/newer', 0), 'one whose schedules could not be moved says so'); + renameDataset('tank/new', 'tank/newer', 0, $cfg), 'one whose schedules could not be moved says so'); zfsmJobUnlock($lock); // Destroy: they go, counts too. $pool('datasets', array("tank\tfilesystem", "tank/new\tfilesystem", "tank/new/sub\tfilesystem", "tank/docker\tfilesystem")); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index bba2fdc3..f94edf4e 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -350,7 +350,7 @@ _(Replicate Every Dataset)_: > - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. If that dataset is copied later, its first copy replaces the empty one and the copies below it stay; an encrypted dataset sent raw can't replace it, nor can any copy replace an empty one that is encrypted, as one under an encrypted dataset is, and the run says what to do. When the copy of the dataset above one is missing because it failed, or the schedule that copies it has not made it yet, that one waits for the next run. > - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - A dataset can have copies of its own, or None, under **Actions**, **Schedules** in the Main tab (**Replication**): it and the datasets below it follow that one instead, unless they have their own, each with its own **Copy Under**. -> - Renaming a dataset in the Main tab renames its copies under each **Copy Under** a schedule uses with it, unless a dataset already has the new name there, no schedule copies it or a dataset below it there now, or what is there is neither its copy nor a holder of the copies below it; it says which of those stayed. Copies under a **Copy Under** no schedule uses now stay as they are. Destroying a dataset removes what it and the datasets below it had under **Actions**, **Schedules**, and leaves its copies. +> - Renaming a dataset in the Main tab renames its copies under each **Copy Under** a schedule uses with it, unless a dataset already has the new name there, no schedule copies it or a dataset below it there now, what is there is neither its copy nor a holder of the copies below it, or a dataset made to hold it would stop an encrypted copy due there; it says which of those stayed, and a clash its schedules meet where they went. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves a copy no room for its bookmark. Copies under a **Copy Under** no schedule uses now stay as they are. Destroying a dataset removes what it and the datasets below it had under **Actions**, **Schedules**, and leaves its copies. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. > - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. A run that falls due while the one before still goes, as a first run can, is skipped and logged; once the one before has gone on for a day, it counts as a problem. > - Each run sends one notification at most: what it copied and its first problems. The first run with a problem sends it; later ones are logged in `/var/log/zfs.master.log` until a run works again, and that run's notification says so. The log has each dataset's replication. diff --git a/zfs.master/backend/ZFSMAdmin.php b/zfs.master/backend/ZFSMAdmin.php index c7eb4422..bed7d2a1 100644 --- a/zfs.master/backend/ZFSMAdmin.php +++ b/zfs.master/backend/ZFSMAdmin.php @@ -177,7 +177,7 @@ function zfsmRequireDestructiveMode($config, $subject) { break; case 'renamedataset': - zfsmReturnAnswer(renameDataset(zfsmPostString('zdataset'), zfsmPostString('newname'), zfsmPostString('force')), 'ZFS Dataset Rename', 'Dataset renamed successfully', 'Unable to rename dataset'); + zfsmReturnAnswer(renameDataset(zfsmPostString('zdataset'), zfsmPostString('newname'), zfsmPostString('force'), $zfsm_cfg), 'ZFS Dataset Rename', 'Dataset renamed successfully', 'Unable to rename dataset'); break; case 'destroydataset': diff --git a/zfs.master/backend/ZFSMOperations.php b/zfs.master/backend/ZFSMOperations.php index cfb29141..79651650 100644 --- a/zfs.master/backend/ZFSMOperations.php +++ b/zfs.master/backend/ZFSMOperations.php @@ -624,13 +624,22 @@ function zfsmSetShareOwner($pool, $names) { return $failed; } -function renameDataset($zdataset, $new_name, $force) { +function renameDataset($zdataset, $new_name, $force, $cfg) { if (!zfsmDatasetExists($zdataset) || !zfsmValidDatasetName($new_name)) return zfsmFailure($new_name, 'Invalid source or destination dataset'); - $args = array('rename'); - if ((string)$force === '1') $args[] = '-f'; - $args[] = $zdataset; - $args[] = $new_name; - return zfsmScheduleOwnFollow(commandAnswer($new_name, runProcess('zfs', $args)), $zdataset, $new_name); + // Held until its copies follow it, so no run copies it under its new name first. + $locks = zfsmReplicateLockTree($zdataset); + if (is_string($locks)) return zfsmFailure($new_name, 'Nothing was renamed: '.$locks); + try { + $why = zfsmScheduleCopiesFit(array_keys($locks), $zdataset, $new_name, $cfg); + if ($why !== '') return zfsmFailure($new_name, 'Nothing was renamed: '.$why); + $args = array('rename'); + if ((string)$force === '1') $args[] = '-f'; + $args[] = $zdataset; + $args[] = $new_name; + return zfsmScheduleOwnFollow(commandAnswer($new_name, runProcess('zfs', $args)), $zdataset, $new_name, $cfg); + } finally { + array_map('zfsmJobUnlock', $locks); + } } function setDatasetProperty($zdataset, $property, $value) { diff --git a/zfs.master/include/ZFSMJobs.php b/zfs.master/include/ZFSMJobs.php index 5dd1aec8..b814c44d 100644 --- a/zfs.master/include/ZFSMJobs.php +++ b/zfs.master/include/ZFSMJobs.php @@ -4,7 +4,11 @@ // One lock file per dataset and job, named by the dataset's guid: the lock follows a renamed dataset, and a long name can't make the file name too long. // The file stays after a run, so two runs never lock different files for one dataset; a crashed run's lock goes with its process. function zfsmJobLockFile($zdataset, $job) { - $guid = getDatasetProperty(explode('/', $zdataset, 2)[0], $zdataset, 'guid'); + return zfsmJobGuidLock(getDatasetProperty(explode('/', $zdataset, 2)[0], $zdataset, 'guid'), $job); +} + +// zfsmJobLockFile of the dataset zfs gave $guid; null when that is no guid. +function zfsmJobGuidLock($guid, $job) { return is_string($guid) && preg_match('/^[0-9]+$/D', $guid) ? zfsmJobDir($job).'/'.$guid.'.lock' : null; } diff --git a/zfs.master/include/ZFSMReplicaPlan.php b/zfs.master/include/ZFSMReplicaPlan.php index a0c85a10..d3587553 100644 --- a/zfs.master/include/ZFSMReplicaPlan.php +++ b/zfs.master/include/ZFSMReplicaPlan.php @@ -186,7 +186,7 @@ function zfsmReplicaPlanHolder($plan, $target) { $where = zfsmReplicaTargetWhere($target, $plan['dest']); if (!zfsmReplicaHolder($target, $plan['dest'])) return $where.' already exists and holds no snapshot, so it can\'t be updated without destroying it'; $held = $where.' only holds the copies of datasets below this one, and '; - $fix = ': move aside (zfs rename) any copy below it that no schedule makes now, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all (a copy a repeat set in Replicate made asks for one Replicate by hand)'; + $fix = ': copy it elsewhere instead, or move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all; a copy destroyed loses the snapshots only it had (one a repeat set in Replicate made asks for one Replicate by hand)'; if ($plan['raw']) return $held.'zfs cannot receive the raw copy of this encrypted dataset over it'.$fix; $crypt = getDatasetProperty('', $plan['dest'], 'encryption'); if ($crypt === null) return $held.'whether it is encrypted could not be read'; diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index e24125b1..8e77d3a0 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -213,6 +213,26 @@ function zfsmReplicateLock($lock_file, $schedule) { return $lock; } +// The replication locks of $zdataset and each dataset below it (zfsmJobLock, no command started holds them), by name; why not when +// one is held or can't be had. +function zfsmReplicateLockTree($zdataset) { + $listed = runProcess('zfs', array('list', '-H', '-o', 'name,guid', '-t', 'filesystem,volume', '-r', $zdataset)); + if (($listed['code'] ?? 1) !== 0) return 'the datasets at and below it could not be read: '.zfsmCommandError($listed, 'zfs list failed'); + $locks = array(); + foreach (zfsmLines($listed['output']) as $line) { + list($name, $guid) = explode("\t", $line, 2) + array(1 => ''); + $file = zfsmJobGuidLock($guid, 'replicate'); + $lock = $file === null ? null : zfsmJobLock($file, false); + if (is_resource($lock)) { + $locks[$name] = $lock; + continue; + } + array_map('zfsmJobUnlock', $locks); + return $lock === false ? 'a replication of '.$name.' is running' : 'the replication lock of '.$name.' could not be opened'; + } + return $locks; +} + // One plan of zfsmRunReplicate's sent, with its log lines and the row's progress: array(0 when it landed, its notification's text and type). // $tag: the copy's (zfsmKeepTag), whose holds each run puts on before its send and a landed one takes off. function zfsmReplicateSent($plan, $zdataset, $snapshot, $take, $target, $tag) { diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index a849d883..b9d2a5fb 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -219,6 +219,26 @@ function zfsmScheduleCopiesPlace($dest, $zdest, $coming, &$here) { return ''; } +// Why renaming $from to $to ($names: it and those below it) would leave a copy a schedule makes of one no room for the bookmark it +// keeps (zfsmReplicaBookmarkFits), where there was; '' when none. +function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { + $read = zfsmScheduleRead(); + // The rename's follow-up says so. + if ($read === null) return ''; + $snapshot = zfsmNewSnapshotName($cfg, time()); + $fits = function($zdataset, $jobs) use ($snapshot) { + $job = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); + return $job === null || !zfsmScheduleRepeats($job['every']) + || zfsmReplicaBookmarkFits($zdataset, zfsmReplicaTargetMark(array('server' => $job['server']), zfsmScheduleCopiesDest($job, $zdataset)), $snapshot); + }; + $after = zfsmScheduleOwnMoved($read['valid'], $from, $to); + foreach ($names as $name) { + $renamed = $to.substr($name, strlen($from)); + if (!$fits($renamed, $after) && $fits($name, $read['valid'])) return 'the names of '.$renamed.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'; + } + return ''; +} + // After zfs renamed $from to $to and its own schedules followed ($before, $after): its copy under each dataset that copies go under // takes $to's name there (zfsmScheduleCopiesMoveOne). Each with a problem, with what. function zfsmScheduleCopiesMove($from, $to, $before, $after) { @@ -231,22 +251,23 @@ function zfsmScheduleCopiesMove($from, $to, $before, $after) { } // The copy of $from under $job's dest, with its bookmarks, renamed to $to's when it is a copy of $to or a holder and a schedule in -// $after still copies $to or one below it there; '' when it moved or is not there, else what went wrong. +// $after still copies $to or one below it there; '' when it moved, is not there, or is not read where none copies it, else what +// went wrong. function zfsmScheduleCopiesMoveOne($job, $from, $to, $after) { $old = zfsmScheduleCopiesDest($job, $from); $new = zfsmScheduleCopiesDest($job, $to); $missing = zfsmDatasetMissing($old); + if (!zfsmScheduleCopiesKept($to, $job['dest'], $after)) return $missing === false ? 'left where it is, as no schedule copies the renamed dataset or one below it there' : ''; // zfs says the same of a copy in a pool not imported now. $pool = $missing === true ? zfsmDatasetMissing(strtok($old, '/')) : false; - if ($pool !== false) return 'not renamed with it, as '.($pool ? 'its pool is not imported now' : 'whether its pool is imported could not be read').'. It stays under its old name, so the next copy of the renamed dataset is a full one'; + if ($pool !== false) return 'not checked, as '.($pool ? 'its pool is not imported now' : 'whether its pool is imported could not be read').': if it is there, it keeps its old name, and the next copy of the renamed dataset is a full one'; if ($missing === true) return ''; if ($missing === null) return 'not renamed with it, as whether it is there could not be read'; - if (!zfsmScheduleCopiesKept($to, $job['dest'], $after)) return 'left where it is, as no schedule copies the renamed dataset or one below it there'; $whose = zfsmScheduleCopiesWhose($old, $to); if ($whose !== '') return $whose; $free = zfsmDatasetMissing($new); if ($free !== true) return $free === null ? 'not renamed with it, as whether its new name is free could not be read' : 'left where it is, as a dataset already has its new name'; - $why = zfsmScheduleCopiesRename($job['dest'], $old, $new); + $why = zfsmScheduleCopiesRename($job['dest'], $old, $new, $after); if ($why !== '') return 'not renamed with it: '.$why.'. It stays under its old name, so the next copy of the renamed dataset is a full one'; $left = implode('; ', array_filter(array(zfsmScheduleCopiesMarks($old, $new, $to), zfsmScheduleCopiesHolds($old, $new, $to)), 'strlen')); return $left === '' ? '' : 'renamed with it, but '.$left; @@ -331,16 +352,46 @@ function zfsmScheduleCopiesKept($zdataset, $dest, $jobs) { } // zfs rename of copy $old to $new under $dest, after a holder is made where a level above $new is missing, as zfs rename needs its -// parent (live, 2.4.4). '' or why not. -function zfsmScheduleCopiesRename($dest, $old, $new) { +// parent (live, 2.4.4), unless that holder would stop a copy $jobs make there (zfsmScheduleCopiesBlocked). '' or why not. +function zfsmScheduleCopiesRename($dest, $old, $new, $jobs) { $here = array(); foreach (zfsmScheduleCopiesLevels($dest, $new) as $level) { $missing = zfsmDatasetMissing($level); if ($missing === null) return 'the datasets above its new name could not be read'; if (!$missing) $here[$level] = true; } + $blocked = zfsmScheduleCopiesBlocked($dest, $new, $here, $jobs); + if ($blocked !== '') return $blocked; $why = zfsmScheduleCopiesPlace($dest, $new, array(), $here); if ($why !== '') return $why; $renamed = runProcess('zfs', array('rename', $old, $new)); return ($renamed['code'] ?? 1) === 0 ? '' : zfsmCommandError($renamed, 'zfs rename failed'); } + +// Of the missing levels above $zdest, a copy under $dest ($here: those there), why a holder at the first would stop a copy $jobs make +// there: zfs receives no raw copy over it, nor any over an encrypted one (zfsmReplicaPlanHolder). '' when none. +function zfsmScheduleCopiesBlocked($dest, $zdest, $here, $jobs) { + $above = substr($dest, 0, -1); + foreach (zfsmScheduleCopiesLevels($dest, $zdest) as $level) { + if (isset($here[$level])) { + $above = $level; + continue; + } + foreach (zfsmScheduleCopiesTo($dest, $level, $jobs) as $zdataset) { + $crypt = array(getDatasetProperty('', $zdataset, 'encryption'), getDatasetProperty('', $above, 'encryption')); + if (in_array(null, $crypt, true)) return 'whether a dataset made at '.$level.' to hold this one would stop the copy due there could not be read'; + if ($crypt !== array('off', 'off')) return 'zfs could not receive the copy due at '.$level.' over the dataset made there to hold this one, as one of the two is encrypted'; + } + } + return ''; +} + +// The datasets $jobs copy to $level under $dest, as zfsmScheduleCopiesComing reads a run's: the one of that name (above the renamed +// one, so there) when a schedule copies it under $dest, and each a repeat set in Replicate sends there. +function zfsmScheduleCopiesTo($dest, $level, $jobs) { + $zdataset = substr($level, strlen($dest)); + $by = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); + $to = ($by['dest'] ?? '') === $dest && zfsmScheduleRepeats($by['every']) ? array($zdataset) : array(); + foreach ($jobs as $job) if (!zfsmScheduleCovers($job) && $job['server'] === '' && $job['dest'] === $level) $to[] = $job['dataset']; + return $to; +} diff --git a/zfs.master/include/ZFSMScheduleDialog.php b/zfs.master/include/ZFSMScheduleDialog.php index eb73ee00..6ecca468 100644 --- a/zfs.master/include/ZFSMScheduleDialog.php +++ b/zfs.master/include/ZFSMScheduleDialog.php @@ -55,12 +55,15 @@ function zfsmScheduleDialogSave($zdataset, $snapshots, $copies, $cfg, $wait = ZF $not = zfsmScheduleDialogWhy($repeats[0] && $check[0] ? zfsmScheduleSnapshotNot($snap['dataset'], $listed) : '', $repeats[1] && $check[1] ? zfsmScheduleCopiesOwnNot($copy, $all, $listed) : ''); if ($not !== '') return 'Nothing was saved: '.$not; + $was = $all; // An hourly part goes in second, so its minute is picked clear of the other's time. if ($snap['every'] === 'hourly') $all = zfsmScheduleSnapshotPlace(zfsmScheduleSnapshotPlace($all, $copy, $drop[1]), $snap, $drop[0]); else $all = zfsmScheduleSnapshotPlace(zfsmScheduleSnapshotPlace($all, $snap, $drop[0]), $copy, $drop[1]); // Both placed, so the pair is checked as saved: by the copies when they are checked. $why = zfsmScheduleDialogWhy($check[0] ? zfsmScheduleSnapshotCheck($all, $snap, $cfg, $check[1] ? array($copy['id'] => 0) : array()) : '', $check[1] ? zfsmScheduleSnapshotCheck($all, $copy, $cfg) : ''); + // A part set to Default hands its datasets to the one above, which may then clash with one as saved. + if ($why === '' && ($drop[0] || $drop[1])) $why = zfsmScheduleNewClash($was, $all, $cfg); return $why === '' ? $all : 'Nothing was saved: '.$why; }, $after); if (!$changed['saved']) return $changed + array('said' => ''); diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index 322bc787..40ef3609 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -204,20 +204,23 @@ function zfsmScheduleSnapshotChange($change, &$after = array()) { return $changed; } -// After zfs renamed $from to $to, or destroyed it ($to ''): the own schedules of both kinds of it and below follow it, or go. As -// zfsmScheduleChange answers, with 'before' and 'after' it (as it would leave them unsaved, null unread); 'why' '' when it has none. +// $jobs once zfs renamed $from to $to, or destroyed it ($to ''): the own schedules of both kinds of it and below follow it, or go. +function zfsmScheduleOwnMoved($jobs, $from, $to) { + $under = function($root) use ($jobs) { return array_filter($jobs, function($job) use ($root) { return zfsmScheduleCovers($job) && zfsmUnderAny($job['dataset'], array($root)); }); }; + $moved = $under($from); + // zfs made $to new: one at or under it is left by a dataset destroyed before, and gives way. + $jobs = array_diff_key($jobs, $moved, $to === '' ? array() : $under($to)); + foreach ($to === '' ? array() : $moved as $id => $job) { + $job['dataset'] = $to.substr($job['dataset'], strlen($from)); + $jobs[$id] = $job; + } + return $jobs; +} + +// zfsmScheduleOwnMoved, saved. As zfsmScheduleChange answers, with 'before' and 'after' it (as it would leave them unsaved, null +// unread); 'why' '' when it has none. function zfsmScheduleOwnMove($from, $to) { - $move = function($jobs) use ($from, $to) { - $under = function($root) use ($jobs) { return array_filter($jobs, function($job) use ($root) { return zfsmScheduleCovers($job) && zfsmUnderAny($job['dataset'], array($root)); }); }; - $moved = $under($from); - // zfs made $to new: one at or under it is left by a dataset destroyed before, and gives way. - $jobs = array_diff_key($jobs, $moved, $to === '' ? array() : $under($to)); - foreach ($to === '' ? array() : $moved as $id => $job) { - $job['dataset'] = $to.substr($job['dataset'], strlen($from)); - $jobs[$id] = $job; - } - return $jobs; - }; + $move = function($jobs) use ($from, $to) { return zfsmScheduleOwnMoved($jobs, $from, $to); }; $read = zfsmScheduleRead(); if ($read !== null && $move($read['valid']) === $read['valid']) return array('saved' => false, 'why' => '', 'before' => $read['valid'], 'after' => $read['valid']); $before = $after = array(); @@ -228,8 +231,9 @@ function zfsmScheduleOwnMove($from, $to) { } // $answer of zfs renaming $from to $to, or destroying it ($to ''), gaining what did not follow when that worked: its own schedules -// (zfsmScheduleOwnMove) and, on a rename only, its copies under each Copy Under (zfsmScheduleCopiesMove). -function zfsmScheduleOwnFollow($answer, $from, $to) { +// (zfsmScheduleOwnMove) and, on a rename only, a clash they meet there ($cfg names the snapshots) and its copies under each Copy +// Under (zfsmScheduleCopiesMove). +function zfsmScheduleOwnFollow($answer, $from, $to, $cfg = array()) { // zfs renames a dataset to its own name by doing nothing. if (!$answer['succeeded'] || $answer['failed'] || $from === $to) return $answer; $moved = zfsmScheduleOwnMove($from, $to); @@ -237,6 +241,8 @@ function zfsmScheduleOwnFollow($answer, $from, $to) { : 'not moved to '.$to.' ('.$moved['why'].'): Remove them in Settings and set them again under Actions, Schedules'; if ($moved['why'] !== '') $answer['failed']['Schedules of '.$from] = $moved['saved'] ? $moved['why'] : $left; if ($to === '') return $answer; + $clash = $moved['saved'] ? zfsmScheduleNewClash($moved['before'], $moved['after'], $cfg) : ''; + if ($clash !== '') $answer['failed']['Schedules of '.$to] = 'moved with it, but '.$clash; if ($moved['before'] === null) { $answer['failed']['Copies of '.$from] = 'not renamed with it, as the schedules that say where they go could not be read'; return $answer; @@ -275,6 +281,16 @@ function zfsmScheduleSnapshotCheck($jobs, $job, $cfg, $aside = array()) { return $clash === '' ? '' : $clash.'; add the time to Snapshots Pattern'; } +// The first clash (zfsmScheduleSnapshotCheck) $after has that $before, the same schedules by id before a change, did not, with the +// schedule it stops; '' when none. One there before stays the owner's to change. +function zfsmScheduleNewClash($before, $after, $cfg) { + foreach (array_intersect_key($after, $before) as $id => $job) { + $why = zfsmScheduleSnapshotCheck($after, $job, $cfg); + if ($why !== '' && zfsmScheduleSnapshotCheck($before, $before[$id], $cfg) === '') return 'for '.zfsmScheduleWhat($job).', '.$why; + } + return ''; +} + // Save's answer: what $job does now, with $jobs as saved; one that runs is counted in $listed. function zfsmScheduleSnapshotSaid($job, $remove, $jobs, $listed) { if ($remove && $job['dataset'] === '') return 'Snapshot Every Dataset is off; the snapshots it took stay'; From 8c7cc8f50ef4e999fe86abf0f1212270b15d59df Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:33:13 +0800 Subject: [PATCH 16/24] fix(schedules): a rename leaves Docker's legacy-mounted layers unlocked unless a repeat copies one; copy checks share one repeat list --- tests/fixtures/bin/zfs | 16 +++++---- tests/schedule_dialog_test.php | 19 ++++++++++ zfs.master/backend/ZFSMOperations.php | 2 +- zfs.master/include/ZFSMReplicate.php | 20 ----------- zfs.master/include/ZFSMScheduleCopies.php | 42 +++++++++++++++++++---- 5 files changed, 64 insertions(+), 35 deletions(-) diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index b474508e..f85b8498 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -102,12 +102,14 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n done exit 0 fi - # A rename's read of a dataset and those below it with their guids (as get reads them); ZFSM_TEST_TREE_FAIL=1 fails it. - if [ "$4" = name,guid ] && [ "$6" = filesystem,volume ] && [ "$7" = -r ]; then + # A rename's read of a dataset and those below it: guid (as get reads it), mountpoint (as the schedule's dataset read); + # ZFSM_TEST_TREE_FAIL=1 fails it. + if [ "$4" = name,guid,mountpoint ] && [ "$6" = filesystem,volume ] && [ "$7" = -r ]; then [ "${ZFSM_TEST_TREE_FAIL:-0}" = 1 ] && { printf "cannot open '%s': Input/output error\n" "$last_arg" >&2; exit 1; } exists "$last_arg" || { printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1; } - awk -F'\t' -v d="$last_arg" '$1 == d || index($1, d "/") == 1 { print $1 }' "$R/datasets" | while IFS= read -r n; do - printf '%s\t%s\n' "$n" "$(prop "$n" guid "$(printf '%s' "$n" | cksum | cut -d' ' -f1)")" + awk -F'\t' -v d="$last_arg" '$1 == d || index($1, d "/") == 1' "$R/datasets" | while IFS="$(printf '\t')" read -r n ty; do + if [ "$ty" = volume ]; then mp=-; else mp=/mnt/$n; fi + printf '%s\t%s\t%s\n' "$n" "$(prop "$n" guid "$(printf '%s' "$n" | cksum | cut -d' ' -f1)")" "$(prop "$n" mountpoint "$mp")" done exit 0 fi @@ -402,9 +404,9 @@ if [ "$1" = "list" ] && contains_arg 'name,mountpoint' "$@" && contains_arg 'fil exit 0 fi -# A rename's read of a dataset and those below it with their guids: the dataset alone, its guid as get gives it. -if [ "$1" = "list" ] && [ "$4" = name,guid ] && [ "$6" = filesystem,volume ] && [ "$7" = -r ]; then - printf '%s\t%s\n' "$last_arg" "${ZFSM_TEST_GUID:-$(printf '%s' "$last_arg" | cksum | cut -d' ' -f1)}" +# A rename's read of a dataset and those below it: the dataset alone, its guid as get gives it. +if [ "$1" = "list" ] && [ "$4" = name,guid,mountpoint ] && [ "$6" = filesystem,volume ] && [ "$7" = -r ]; then + printf '%s\t%s\t/mnt/%s\n' "$last_arg" "${ZFSM_TEST_GUID:-$(printf '%s' "$last_arg" | cksum | cut -d' ' -f1)}" "$last_arg" exit 0 fi diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index 34b960f6..d423a427 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -322,6 +322,8 @@ function same($expected, $actual, $message) { same(true, $save('tank/j', $part('default'), $part('days', '05:00', '1', 0, $far))['saved'], 'own copies under a long name'); same(array(array('tank/jj' => 'Nothing was renamed: the names of tank/jj/'.$kin.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'), true), array(renameDataset('tank/j', 'tank/jj', 0, $cfg)['failed'], $save('tank/j', $part('default'), $part('default'))['saved']), 'the room under its own Copy Under'); +same(array(true, array('tank/k'.str_repeat('x', 10) => 0)), array($save('tank/k', $part('default'), $part('none'))['saved'], renameDataset('tank/k', 'tank/k'.str_repeat('x', 10), 0, $cfg)['succeeded']), + 'none needed where no copy is made'); // A replication of it or one below it running: nothing is renamed, and the locks it took are let go. $add(array('tank/lk', 'tank/lk/kid')); $lock = zfsmJobLock(zfsmJobLockFile('tank/lk/kid', 'replicate'), false); @@ -331,6 +333,23 @@ function same($expected, $actual, $message) { same(array(array('succeeded' => array(), 'failed' => array('tank/lk2' => 'Nothing was renamed: a replication of tank/lk/kid is running')), true, true), array($busy, is_resource($loose), in_array('tank/lk/kid', $datasets(), true)), 'nothing renamed while one below it copies, and its lock let go'); zfsmJobUnlock($loose); +// Docker's legacy-mounted image layers, which no run copies: not held, so not waited for. A read-only one and a legacy one a +// repeat set in Replicate copies are. +$add(array('tank/lk/layer', 'tank/lk/ro', 'tank/lk/leg')); +$pool('props', array_merge($read('pool/props'), array("tank/lk/layer\tmountpoint\tlegacy", "tank/lk/leg\tmountpoint\tlegacy", "tank/lk/ro\treadonly\ton"))); +same(true, zfsmScheduleChange(function($jobs) { return $jobs + array('6f8b2c3d' => array('id' => '6f8b2c3d', 'dataset' => 'tank/lk/leg', 'dest' => 'backup/leg', 'server' => '', 'every' => 'daily', 'hour' => 3, 'day' => 0)); })['saved'], + 'a repeat set in Replicate of a legacy-mounted one'); +$past = array(); +foreach (array('tank/lk/layer', 'tank/lk/ro', 'tank/lk/leg') as $below) { + $held = zfsmJobLock(zfsmJobLockFile($below, 'replicate'), false); + $past[] = renameDataset('tank/lk', 'tank/lk3', 0, $cfg); + zfsmJobUnlock($held); + if ($past[count($past) - 1]['succeeded']) renameDataset('tank/lk3', 'tank/lk', 0, $cfg); +} +$running = function($zdataset) { return array('succeeded' => array(), 'failed' => array('tank/lk3' => 'Nothing was renamed: a replication of '.$zdataset.' is running')); }; +same(array(array('tank/lk3' => 0), $running('tank/lk/ro'), $running('tank/lk/leg')), array($past[0]['succeeded'], $past[1], $past[2]), + 'a legacy-mounted one no run copies is not held; a read-only one, and one a repeat copies, are'); +same(true, zfsmScheduleChange(function($jobs) { unset($jobs['6f8b2c3d']); return $jobs; })['saved'], 'the repeat removed'); putenv('ZFSM_TEST_TREE_FAIL=1'); $unread = renameDataset('tank/lk', 'tank/lk2', 0, $cfg)['failed']; putenv('ZFSM_TEST_TREE_FAIL'); diff --git a/zfs.master/backend/ZFSMOperations.php b/zfs.master/backend/ZFSMOperations.php index 79651650..42008a6d 100644 --- a/zfs.master/backend/ZFSMOperations.php +++ b/zfs.master/backend/ZFSMOperations.php @@ -627,7 +627,7 @@ function zfsmSetShareOwner($pool, $names) { function renameDataset($zdataset, $new_name, $force, $cfg) { if (!zfsmDatasetExists($zdataset) || !zfsmValidDatasetName($new_name)) return zfsmFailure($new_name, 'Invalid source or destination dataset'); // Held until its copies follow it, so no run copies it under its new name first. - $locks = zfsmReplicateLockTree($zdataset); + $locks = zfsmScheduleCopiesHold($zdataset); if (is_string($locks)) return zfsmFailure($new_name, 'Nothing was renamed: '.$locks); try { $why = zfsmScheduleCopiesFit(array_keys($locks), $zdataset, $new_name, $cfg); diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index 8e77d3a0..e24125b1 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -213,26 +213,6 @@ function zfsmReplicateLock($lock_file, $schedule) { return $lock; } -// The replication locks of $zdataset and each dataset below it (zfsmJobLock, no command started holds them), by name; why not when -// one is held or can't be had. -function zfsmReplicateLockTree($zdataset) { - $listed = runProcess('zfs', array('list', '-H', '-o', 'name,guid', '-t', 'filesystem,volume', '-r', $zdataset)); - if (($listed['code'] ?? 1) !== 0) return 'the datasets at and below it could not be read: '.zfsmCommandError($listed, 'zfs list failed'); - $locks = array(); - foreach (zfsmLines($listed['output']) as $line) { - list($name, $guid) = explode("\t", $line, 2) + array(1 => ''); - $file = zfsmJobGuidLock($guid, 'replicate'); - $lock = $file === null ? null : zfsmJobLock($file, false); - if (is_resource($lock)) { - $locks[$name] = $lock; - continue; - } - array_map('zfsmJobUnlock', $locks); - return $lock === false ? 'a replication of '.$name.' is running' : 'the replication lock of '.$name.' could not be opened'; - } - return $locks; -} - // One plan of zfsmRunReplicate's sent, with its log lines and the row's progress: array(0 when it landed, its notification's text and type). // $tag: the copy's (zfsmKeepTag), whose holds each run puts on before its send and a landed one takes off. function zfsmReplicateSent($plan, $zdataset, $snapshot, $take, $target, $tag) { diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index b9d2a5fb..7ad3718d 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -169,12 +169,17 @@ function zfsmScheduleCopiesComing($job, $jobs, $listed) { $by = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); if (($by['dest'] ?? '') === $job['dest'] && zfsmScheduleLeftOut($by, $jobs, $zdataset, $row, $listed['name']) === '') $coming[zfsmScheduleCopiesDest($by, $zdataset)] = true; } - foreach ($jobs as $other) { - if (!zfsmScheduleCovers($other) && $other['server'] === '') $coming[$other['dest']] = true; - } + foreach (zfsmScheduleCopiesRepeatsTo($jobs) as $zdest => $from) $coming[$zdest] = true; return $coming; } +// The datasets each repeat set in Replicate copies on this server, by the copy it makes. +function zfsmScheduleCopiesRepeatsTo($jobs) { + $to = array(); + foreach ($jobs as $job) if (!zfsmScheduleCovers($job) && $job['server'] === '') $to[$job['dest']][] = $job['dataset']; + return $to; +} + // One dataset's copy (zfsmRunReplicate): array(its code, array(its text, type)); an error ends only this one. $wait: what the run has // left to wait, less what this one waits. function zfsmScheduleCopiesOne($job, $zdataset, $zdest, $name, &$wait) { @@ -239,6 +244,31 @@ function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { return ''; } +// The replication locks (zfsmJobLock, no command started holds them) of $zdataset and each dataset below it a run can copy, by +// name; why not when one is held or can't be had. +function zfsmScheduleCopiesHold($zdataset) { + $listed = runProcess('zfs', array('list', '-H', '-o', 'name,guid,mountpoint', '-t', 'filesystem,volume', '-r', $zdataset)); + if (($listed['code'] ?? 1) !== 0) return 'the datasets at and below it could not be read: '.zfsmCommandError($listed, 'zfs list failed'); + $read = zfsmScheduleRead(); + $repeats = array(); + foreach ($read === null ? array() : $read['valid'] as $job) if (!zfsmScheduleCovers($job)) $repeats[$job['dataset']] = true; + $locks = array(); + foreach (zfsmLines($listed['output']) as $line) { + list($name, $guid, $mountpoint) = explode("\t", $line, 3) + array('', '', ''); + // Docker's legacy-mounted image layers, often thousands: a lock each could run out of open files. Copy Under skips them. + if ($mountpoint === 'legacy' && !isset($repeats[$name])) continue; + $file = zfsmJobGuidLock($guid, 'replicate'); + $lock = $file === null ? null : zfsmJobLock($file, false); + if (is_resource($lock)) { + $locks[$name] = $lock; + continue; + } + array_map('zfsmJobUnlock', $locks); + return $lock === false ? 'a replication of '.$name.' is running' : 'the replication lock of '.$name.' could not be opened'; + } + return $locks; +} + // After zfs renamed $from to $to and its own schedules followed ($before, $after): its copy under each dataset that copies go under // takes $to's name there (zfsmScheduleCopiesMoveOne). Each with a problem, with what. function zfsmScheduleCopiesMove($from, $to, $before, $after) { @@ -390,8 +420,6 @@ function zfsmScheduleCopiesBlocked($dest, $zdest, $here, $jobs) { // one, so there) when a schedule copies it under $dest, and each a repeat set in Replicate sends there. function zfsmScheduleCopiesTo($dest, $level, $jobs) { $zdataset = substr($level, strlen($dest)); - $by = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); - $to = ($by['dest'] ?? '') === $dest && zfsmScheduleRepeats($by['every']) ? array($zdataset) : array(); - foreach ($jobs as $job) if (!zfsmScheduleCovers($job) && $job['server'] === '' && $job['dest'] === $level) $to[] = $job['dataset']; - return $to; + $own = (zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder')['dest'] ?? '') === $dest ? array($zdataset) : array(); + return array_merge($own, zfsmScheduleCopiesRepeatsTo($jobs)[$level] ?? array()); } From 50ca9de410d319b1d7a4b8568fe4733ac2b5c624 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:30:53 +0800 Subject: [PATCH 17/24] fix(schedules): a rename needs Destructive Mode and waits for a copy of a legacy layer; Remove checks the clash Default makes; no copy is received over a mounted one - Rename is refused server-side without Destructive Mode, as destroy is. - A rename checks the lock of a legacy-mounted dataset where a run made one, so a one-off Replicate of it stops the rename. - The bookmark room check skips a dataset no run copies: in the copy's pool, left to Sanoid, read-only or legacy-mounted. - A holder is made under an encrypted parent when no run copies that parent there. - A clash check counts a schedule's changed clash as new; Remove in Settings runs it. - A copy is not received over a holder while a copy below it is mounted. - A run's notification gives the last step's problem first, then the first step's warning. - The fake zfs keeps receive's -o properties and lists -r below a dataset. --- README.md | 2 +- tests/admin_gate_test.php | 1 + tests/fixtures/bin/zfs | 27 ++++-- tests/keep_test.php | 4 +- tests/run_http_inventory_tests.sh | 2 +- tests/schedule_copies_test.php | 29 +++++-- tests/schedule_dialog_test.php | 91 ++++++++++++++++++--- tests/schedule_test.php | 10 +-- zfs.master/ZFSMasterSettings.page | 6 +- zfs.master/backend/ZFSMAdmin.php | 6 +- zfs.master/include/ZFSMReplicaPlan.php | 5 +- zfs.master/include/ZFSMReplicaTarget.php | 11 +++ zfs.master/include/ZFSMReplicate.php | 7 +- zfs.master/include/ZFSMSchedule.php | 12 +-- zfs.master/include/ZFSMScheduleCopies.php | 68 +++++++++------ zfs.master/include/ZFSMScheduleSnapshot.php | 6 +- 16 files changed, 213 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 34d2e037..1477579c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ See and manage your ZFS pools, datasets and snapshots from Unraid's **Main** tab Each dataset row has an **Actions** menu. These items do more than their name says: - **Replicate** — copies the dataset to another pool, or to another ZFS Master NG server added in Settings. Run it once, or repeat it hourly or on chosen days and keep the newest snapshots, plus the newest of each of the last days, weeks, months or years you choose. New sends keep records larger than 128K whole; what a copy already holds stays as it is. A copy of an unencrypted dataset to a server travels compressed with zstd when both servers have it. -- **Schedules** — gives the dataset its own snapshot schedule (Snapshots) and its own copies into another pool (Replication), in place of Snapshot Every Dataset and Replicate Every Dataset (Settings). Each part is set on its own: Default follows the nearest dataset above with its own, else Settings; None stops that part only, so Replication None stops the copies under Copy Under while Snapshots on Default still takes snapshots, and a repeat set in Replicate still runs. Datasets below it follow it unless they have their own. Renaming the dataset renames its copies under Copy Under too, where a schedule still copies it or a dataset below it there, what is there is its copy (or holds the copies below it), and its new name is free; the rename says which stayed. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves a copy no room for its bookmark. +- **Schedules** — gives the dataset its own snapshot schedule (Snapshots) and its own copies into another pool (Replication), in place of Snapshot Every Dataset and Replicate Every Dataset (Settings). Each part is set on its own: Default follows the nearest dataset above with its own, else Settings; None stops that part only, so Replication None stops the copies under Copy Under while Snapshots on Default still takes snapshots, and a repeat set in Replicate still runs. Datasets below it follow it unless they have their own. Renaming the dataset renames its copies under Copy Under too, where a schedule still copies it or a dataset below it there, what is there is its copy (or holds the copies below it), its new name is free, and no dataset made to hold it would stop an encrypted copy due there; the rename says which stayed. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves no room for the bookmark a copy made now keeps (named by Snapshots Pattern as it is now); a bookmark made before that no longer fits stays, and the rename says so. - **Rewrite Files** (Unraid 7.3+) — writes the dataset's files again, so a changed compression, checksum, dedup or copies setting applies to data already there. Snapshots keep the old blocks, so the pool can use more space until they are deleted. - **Snapshots Admin** — lists the dataset's snapshots, and its children's if you like, to filter and sort. From there you can browse a snapshot's files, see what changed since it, roll back, clone, hold, rename, or delete several at once. - **Promote Dataset** (clones only) — makes a clone stand on its own, no longer tied to the snapshot it was cloned from. The original dataset can then be destroyed and the clone stays. diff --git a/tests/admin_gate_test.php b/tests/admin_gate_test.php index de4475d2..8b29bb2e 100644 --- a/tests/admin_gate_test.php +++ b/tests/admin_gate_test.php @@ -13,6 +13,7 @@ 'rollback_newer' => array(array('cmd' => 'rollbacksnapshot', 'snapshot' => 'tank/data@daily', 'destroy_newer' => '1'), '0', '', 'blocked'), 'destroy_recursive' => array(array('cmd' => 'destroysnapshot', 'snapshot' => 'tank/data@daily', 'recursive' => '1'), '0', '', 'blocked'), 'destroy_dataset' => array(array('cmd' => 'destroydataset', 'zdataset' => 'tank/data', 'force' => '0'), '0', '', 'blocked'), + 'rename_dataset' => array(array('cmd' => 'renamedataset', 'zdataset' => 'tank/data', 'newname' => 'tank/data2', 'force' => '0'), '0', '', 'blocked'), 'rollback_plain' => array(array('cmd' => 'rollbacksnapshot', 'snapshot' => 'tank/data@daily', 'destroy_newer' => '0'), '0', '', 'allowed'), 'destroy_plain' => array(array('cmd' => 'destroysnapshot', 'snapshot' => 'tank/data@daily', 'recursive' => '0'), '0', '', 'allowed'), 'newer_with_mode' => array(array('cmd' => 'rollbacksnapshot', 'snapshot' => 'tank/data@daily', 'destroy_newer' => '1'), '1', '', 'allowed'), diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index f85b8498..064a50f6 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -39,24 +39,27 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n if contains_arg 'name,readonly,mountpoint' "$@"; then [ "${ZFSM_TEST_COVER_FAIL:-0}" = 1 ] && { printf 'cannot iterate filesystems: Input/output error\n' >&2; exit 1; } [ "${ZFSM_TEST_COVER_FAIL:-0}" = hang ] && { sleep 30; exit 1; } - code=0; under='' + code=0; under=''; deep=0 contains_arg '-d' "$@" && under=$last_arg + contains_arg '-r' "$@" && { under=$last_arg; deep=1; } if [ -n "$under" ] && ! exists "$under"; then printf "cannot open '%s': dataset does not exist\n" "$under" >&2; exit 1; fi while IFS="$(printf '\t')" read -r n ty; do - if [ -n "$under" ]; then case "$n" in "$under") ;; "$under"/*/*) continue ;; "$under"/*) ;; *) continue ;; esac; fi + if [ -n "$under" ]; then case "$n" in "$under") ;; "$under"/*/*) [ "$deep" = 1 ] || continue ;; "$under"/*) ;; *) continue ;; esac; fi [ "$n" = "${ZFSM_TEST_COVER_FAIL:-}" ] && { printf "cannot open '%s': Input/output error\n" "$n" >&2; code=1; continue; } if [ "$ty" = volume ]; then mp=-; else mp=/mnt/$n; fi printf '%s\t%s\t%s\n' "$n" "$(prop "$n" readonly off)" "$(prop "$n" mountpoint "$mp")" done < "$R/datasets" exit $code fi - # And its read of snapshot names alone, sorted; with -d 1 only that one's. ZFSM_TEST_NAMES_FAIL=1 fails it, and - # ZFSM_TEST_COVER_FAIL's dataset fails here too, as zfs does. + # And its read of snapshot names alone, sorted; with -d 1 only that one's, with -r its and below. ZFSM_TEST_NAMES_FAIL=1 + # fails it, and ZFSM_TEST_COVER_FAIL's dataset fails here too, as zfs does. if [ "$4" = name ] && [ "$5" = -s ] && [ "$8" = snapshot ]; then [ "${ZFSM_TEST_NAMES_FAIL:-0}" = 1 ] && { printf 'cannot iterate filesystems: Input/output error\n' >&2; exit 1; } - under=''; contains_arg '-d' "$@" && under=$last_arg - f=${ZFSM_TEST_COVER_FAIL:-}; case $f in 1|hang) f='' ;; esac; [ -n "$under" ] && [ "$under" != "$f" ] && f='' - awk -F'\t' -v d="$under" -v f="$f" '((d == "" && index($1, "@")) || (d != "" && index($1, d "@") == 1)) && index($1, f "@") != 1 { print $1 }' "$R/marks" | LC_ALL=C sort + under=''; deep=0; contains_arg '-d' "$@" && under=$last_arg + contains_arg '-r' "$@" && { under=$last_arg; deep=1; } + f=${ZFSM_TEST_COVER_FAIL:-}; case $f in 1|hang) f='' ;; esac + if [ -n "$under" ] && [ "$under" != "$f" ]; then case "$deep:$f" in 1:"$under"/*) ;; *) f='' ;; esac; fi + awk -F'\t' -v d="$under" -v r="$deep" -v f="$f" '((d == "" && index($1, "@")) || (d != "" && (index($1, d "@") == 1 || (r && index($1, d "/") == 1 && index($1, "@"))))) && index($1, f "@") != 1 { print $1 }' "$R/marks" | LC_ALL=C sort [ -z "$f" ] || { printf "cannot open '%s': Input/output error\n" "$f" >&2; exit 1; } exit 0 fi @@ -88,6 +91,14 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n exists "$last_arg" && { awk -F'\t' -v n="$last_arg" '$1 == n { print $1 "\t" $2 }' "$R/datasets"; exit 0; } printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1 fi + # A holder's filesystems and below it, mounted as props say, else no (a copy is received unmounted); ZFSM_TEST_MOUNTED_FAIL=1 fails it. + if [ "$4" = name,mounted ] && [ "$6" = filesystem ] && [ "$7" = -r ]; then + [ "${ZFSM_TEST_MOUNTED_FAIL:-0}" = 1 ] && { printf "cannot open '%s': Input/output error\n" "$last_arg" >&2; exit 1; } + awk -F'\t' -v d="$last_arg" '($1 == d || index($1, d "/") == 1) && $2 == "filesystem" { print $1 }' "$R/datasets" | while IFS= read -r n; do + printf '%s\t%s\n' "$n" "$(prop "$n" mounted no)" + done + exit 0 + fi # The bookmarks of a dataset and below; ZFSM_TEST_BOOKMARKS_FAIL=1 fails the read. if [ "$4" = name,guid ] && [ "$6" = bookmark ] && [ "$7" = -r ]; then [ "${ZFSM_TEST_BOOKMARKS_FAIL:-0}" = 1 ] && { printf "cannot open '%s': Input/output error\n" "$last_arg" >&2; exit 1; } @@ -207,6 +218,8 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n printf 'cannot receive incremental stream: most recent snapshot of %s does not match incremental source\n' "$dest" >&2; exit 1 fi exists "$dest" || printf '%s\t%s\n' "$dest" "$(awk -F'\t' -v n="${snap%@*}" '$1 == n { print $2 }' "$R/datasets")" >> "$R/datasets" + # Each -o prop=value lands on the copy, as readonly=on keeps a run from snapshotting it. + o=0; for a in "$@"; do [ "$o" = 1 ] && set_prop "$dest" "${a%%=*}" "${a#*=}"; o=0; [ "$a" = -o ] && o=1; done if ! printf '%s\n' "$stream" | grep -q '^ZFSMEND$'; then set_prop "$dest" receive_resume_token "tok:$snap" printf 'tok:%s\t%s\n' "$snap" "$guid" >> "$R/tokens" diff --git a/tests/keep_test.php b/tests/keep_test.php index 8bfa0d29..8da3affa 100644 --- a/tests/keep_test.php +++ b/tests/keep_test.php @@ -326,7 +326,7 @@ function same($expected, $actual, $message) { $tagged = function() use ($tag) { return count(array_filter(zfsmKeepSnapshots('tank/data'), function($snapshot) use ($tag) { return $snapshot['tag'] === $tag; })); }; $run(19); $run(20); -same(array(2, true), array($tagged(), zfsmScheduleRemove($id)['saved']), 'two of the schedule\'s snapshots, and it is removed'); +same(array(2, true), array($tagged(), zfsmScheduleRemove($id, $cfg)['saved']), 'two of the schedule\'s snapshots, and it is removed'); $reset(); $add('tank/data', '', 'backup/data', '', 'hourly', '', '', 1, $cfg); $again = $stored()[0]['id']; @@ -335,7 +335,7 @@ function same($expected, $actual, $message) { same(array(0, 6, 1), array(zfsmScheduleRun($again, $cfg, (int)($t0 + 3600 * 21)), count($destroys()), $tagged()), 'the next keeps the newest 1, removing the rest here and in the copy'); // Rules go through the store and a run: Repeat saves them as text in one order; 1 1d keeps only the newest (the newest day's // newest is that one too). -same(true, zfsmScheduleRemove($again)['saved'], 'that schedule is removed'); +same(true, zfsmScheduleRemove($again, $cfg)['saved'], 'that schedule is removed'); $reset(); $add('tank/data', '', 'backup/data', '', 'hourly', '', '', zfsmKeepParse(' 1D, 1 '), $cfg); $jobs = $stored(); diff --git a/tests/run_http_inventory_tests.sh b/tests/run_http_inventory_tests.sh index 6adf911b..ac12658a 100644 --- a/tests/run_http_inventory_tests.sh +++ b/tests/run_http_inventory_tests.sh @@ -33,7 +33,7 @@ for mode in full lazy pool invalid missing gone pool_failure snapshot_failure re done "$zfsm_php" -n "$zfsm_repo/tests/inventory_timeout_test.php" ZFSM_TEST_LOG=/tmp/zfsm-backend-commands.log "$zfsm_php" -n ${ZFSM_TEST_POSIX:+-d "extension=$ZFSM_TEST_POSIX"} "$zfsm_repo/tests/backend_logic_test.php" -for gate in rollback_newer destroy_recursive destroy_dataset rollback_plain destroy_plain destroy_unreviewed snapshot_route_dataset newer_with_mode invalid_exclusion lua_exclusion encrypt_no_passphrase encrypt_other_value encrypt_inherit_passphrase preview_destroy preview_rollback diff_snapshot preview_convert accept_inside_pool accept_pool_top accept_padded accept_missing accept_unread keep_read keep_unread; do +for gate in rollback_newer destroy_recursive destroy_dataset rename_dataset rollback_plain destroy_plain destroy_unreviewed snapshot_route_dataset newer_with_mode invalid_exclusion lua_exclusion encrypt_no_passphrase encrypt_other_value encrypt_inherit_passphrase preview_destroy preview_rollback diff_snapshot preview_convert accept_inside_pool accept_pool_top accept_padded accept_missing accept_unread keep_read keep_unread; do "$zfsm_php" -n "$zfsm_repo/tests/admin_gate_test.php" "$gate" done "$zfsm_php" -n "$zfsm_repo/tests/impact_test.php" diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 7949dcb5..ef1dd055 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -206,6 +206,16 @@ function same($expected, $actual, $message) { putenv('ZFSM_TEST_CRYPT_FAIL=backup/tank/sealed'); same(array('can_proceed' => false, 'error' => 'backup/tank/sealed only holds the copies of datasets below this one, and whether it is encrypted could not be read'), $plan(), 'one whose encryption could not be read'); putenv('ZFSM_TEST_CRYPT_FAIL'); +// Nor while a copy below it is mounted, which receiving over it would unmount (live, 2.4.4); one not read is not sent over. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/mnt\tfilesystem", "backup/tank/mnt\tfilesystem", "backup/tank/mnt/kid\tfilesystem"))); +$pool('marks', array_merge($read('pool/marks'), array("tank/mnt@a\t4747\t9005"))); +$pool('props', array_merge($read('pool/props'), array("backup/tank/mnt\tzfs.master:holder\ton", "backup/tank/mnt/kid\tmounted\tyes"))); +$plan = function() { return zfsmReplicatePlan('tank/mnt', 'a', 'backup/tank/mnt', array('estimate' => false)); }; +$over = 'backup/tank/mnt only holds the copies of datasets below this one, and '; +same(array('can_proceed' => false, 'error' => $over.'backup/tank/mnt/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted (zfs unmount backup/tank/mnt/kid), the next run sends it'), $plan(), 'one with a copy below it mounted'); +putenv('ZFSM_TEST_MOUNTED_FAIL=1'); +same(array('can_proceed' => false, 'error' => $over.'whether a copy below it is mounted could not be read: cannot open \'backup/tank/mnt\': Input/output error'), $plan(), 'one where that could not be read'); +putenv('ZFSM_TEST_MOUNTED_FAIL'); // A first copy onto a holder that stops part-way carries on only with -F again (OpenZFS 2.4.4); the mark goes once it lands. $pool('datasets', array_merge($read('pool/datasets'), array("tank/res\tfilesystem", "backup/tank/res\tfilesystem"))); @@ -216,6 +226,11 @@ function same($expected, $actual, $message) { same(1, zfsmRunReplicate('tank/res', 'a', 'backup/tank/res'), 'a first copy onto a holder, cut off'); putenv('ZFSM_TEST_SEND_CUT'); same(array(true, 'resume'), array(zfsmReplicaHolder(array('server' => ''), 'backup/tank/res'), zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false))['mode']), 'still a holder, part-received'); +$pool('datasets', array_merge($read('pool/datasets'), array("backup/tank/res/kid\tfilesystem"))); +$pool('props', array_merge($read('pool/props'), array("backup/tank/res/kid\tmounted\tyes"))); +same(array('can_proceed' => false, 'error' => 'backup/tank/res only holds the copies of datasets below this one, and backup/tank/res/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted (zfs unmount backup/tank/res/kid), the next run sends it'), + zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false)), 'nor carried on over a copy below it mounted'); +$pool('props', array_values(preg_grep("~^backup/tank/res/kid\tmounted\t~", $read('pool/props'), PREG_GREP_INVERT))); putenv('ZFSM_TEST_MARKS_FAIL=backup/tank/res'); same(array('can_proceed' => false, 'error' => 'The snapshots of backup/tank/res could not be read'), zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false)), 'not carried on when its snapshots could not be read'); putenv('ZFSM_TEST_MARKS_FAIL'); @@ -247,14 +262,15 @@ function same($expected, $actual, $message) { same(array("tank/data@$p\t$hold", "tank/data@$o\t$hold"), array_values(preg_grep('~^tank/data@~', $read('pool/holds'))), 'a run cut off holds for the copy'); same(0, zfsmRunReplicate('tank/data', $p, 'backup/tank/data'), 'carried on by hand'); same(array(), preg_grep('~^tank/data@~', $read('pool/holds')), 'which takes its holds off'); -// The next run carries the others on, then sends what it took: a warning of the first step is told with the second. +// The next run carries the others on, then sends what it took: a warning of the second step is told whole, then the first's, +// which a notification cut short loses. $reset(); @unlink(zfsmScheduleStateFile($id)); putenv('ZFSM_TEST_BOOKMARK_FAIL=1'); zfsmScheduleRun($id, $cfg, $t + 13500); putenv('ZFSM_TEST_BOOKMARK_FAIL'); -same(1, preg_match('/^warning: .*'.preg_quote('tank@'.$p, '/').' to backup\/tank finished \(carried on from where it stopped\)\. The bookmark that keeps the next run incremental could not be made: .*?\. Replication of '.preg_quote('tank@'.$name($t + 13500), '/').' to backup\/tank finished \(changes since /', - implode("\n", $read('notify-detail.log'))), 'the first step\'s warning, then the second\'s: '.implode("\n", $read('notify-detail.log'))); +same(1, preg_match('/^warning: .*'.preg_quote('tank@'.$name($t + 13500), '/').' to backup\/tank finished \(changes since \S+\)\. The bookmark that keeps the next run incremental could not be made: [^.]*permission denied\. Before it: Replication of '.preg_quote('tank@'.$p, '/').' to backup\/tank finished \(carried on from where it stopped\)/', + implode("\n", $read('notify-detail.log'))), 'the second step\'s warning whole, then the first\'s: '.implode("\n", $read('notify-detail.log'))); // A first copy onto a holder, cut off, then carried on: a mark that stays is a problem of the run, though the next step lands cleanly // (the notification's cut ends the line before that step). $pool('datasets', array_merge($read('pool/datasets'), array("tank/ahres\tfilesystem", "backup/tank/ahres\tfilesystem"))); @@ -281,8 +297,8 @@ function same($expected, $actual, $message) { $code = zfsmRunReplicate('tank/data', 'cut2', 'backup/tank/data', null, true, '', $by); putenv('ZFSM_TEST_BOOKMARK_FAIL'); putenv('ZFSM_TEST_MARKS_FAIL'); -same(array(1, 1), array($code, preg_match('/^warning: Replication of tank\/data@cut1 to backup\/tank\/data finished \(carried on from where it stopped\)\. The bookmark that keeps the next run incremental could not be made: .*\. Replication of tank\/data to backup\/tank\/data not started: The snapshots of backup\/tank\/data could not be read$/', implode("\n", $told))), - 'the first step\'s warning, then why the next was not started: '.implode("\n", $told)); +same(array(1, 1), array($code, preg_match('/^\w+: Replication of tank\/data to backup\/tank\/data not started: The snapshots of backup\/tank\/data could not be read\. Before it: Replication of tank\/data@cut1 to backup\/tank\/data finished \(carried on from where it stopped\)\. The bookmark that keeps the next run incremental could not be made: .*$/', implode("\n", $told))), + 'why the next was not started, then the first step\'s warning: '.implode("\n", $told)); // Other replications of tank/data and tank/docker still running, after tank's send outlasted the budget: only waits use it, and // they share it, so the second has none left. @@ -372,7 +388,8 @@ function same($expected, $actual, $message) { file_put_contents(zfsmScheduleFile(), json_encode(array(array('keep' => 1) + $every, array('id' => 'dddd0000', 'dataset' => 'tank/data', 'dest' => '/', 'every' => 'none') + $every, $under))); $reset(); same(array(1, array(), array()), array(zfsmScheduleRun('cccc0000', $cfg, $t + 32400), preg_grep('~^backup/~', $datasets()), preg_grep('/^create /', $read('calls.log'))), 'no holder where Replicate Every Dataset copies'); -same(array(0, 0, array('backup/tank', 'backup/tank/data', 'backup/tank/data/sub')), array(zfsmScheduleRun($id, $cfg, $t + 36000), zfsmScheduleRun('cccc0000', $cfg, $t + 36060), array_values(preg_grep('~^backup/~', $datasets()))), +same(array(0, 0, array('backup/tank', 'backup/tank/data', 'backup/tank/data/sub'), array('create -o canmount=off -o readonly=on -o zfs.master:holder=on backup/tank/data')), + array(zfsmScheduleRun($id, $cfg, $t + 36000), zfsmScheduleRun('cccc0000', $cfg, $t + 36060), array_values(preg_grep('~^backup/~', $datasets())), array_values(preg_grep('/^create /', $read('calls.log')))), 'its copy, then a holder for tank/data, which nothing copies, and the copy below'); echo "schedule copies tests passed\n"; diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index d423a427..c6053ad0 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -168,6 +168,19 @@ function same($expected, $actual, $message) { preg_match('/^moved with it, but for (the copies of tank\/apps\/mv and below|the snapshot schedule of tank\/apps), Snapshots Pattern would give the run at .* the same snapshot name as .*; add the time to Snapshots Pattern$/', $moved['failed']['Schedules of tank/apps/mv'] ?? ''), array_values(preg_grep('~^tank/(apps/)?mv$~', array_column($stored(), 'dataset')))), 'a rename into a clash says it, and its schedule moves'); +// Remove in Settings sets a dataset's own back to Default too: checked as that is. +$own = array_values(array_filter($stored(), function($job) { return $job['dataset'] === 'tank/apps/db' && zfsmScheduleSnapshotsOnly($job); }))[0]['id']; +$removed = array(zfsmScheduleRemove($own, array('snap_pattern' => 'Ymd') + $cfg)); +$removed[] = in_array($own, array_column($stored(), 'id'), true); +$removed[] = zfsmScheduleRemove($own, $cfg)['saved']; +same(array(array('why' => 1, 'saved' => false), true, true), array(array('why' => preg_match('/^Not removed: for (the copies of tank\/apps\/db and below|the snapshot schedule of tank\/apps), Snapshots Pattern would give the run at .* the same snapshot name as .*; add the time to Snapshots Pattern$/', + $removed[0]['why'])) + $removed[0], $removed[1], $removed[2]), 'Remove in Settings is refused where Default would clash, and allowed where not'); +// A clash of each schedule there before (a pattern of months, so each run of the week names alike) does not hide a new one. +$ym = function($zdataset, $snapshots, $copies) use ($cfg) { return zfsmScheduleDialogSave($zdataset, $snapshots, $copies, array('snap_pattern' => 'Ym') + $cfg); }; +same(array(true, 1), array($save('tank/apps/log', $part('none'), $part('days', '03:00', '1', 0, 'usb'))['saved'], + preg_match('/^Nothing was saved: for (the copies of tank\/apps\/log and below|the snapshot schedule of tank\/apps), the .* starts then too; pick another time$/', $ym('tank/apps/log', $part('default'), $part('days', '03:00', '1', 0, 'usb'))['why'])), + 'a clash in its minute, new where each had a clash of names'); +same(true, $save('tank/apps/log', $part('default'), $part('default'))['saved'], 'and back'); same(array(true, true, true), array_map(function($zdataset) use ($save, $part) { return $save($zdataset, $part('default'), $part('default'))['saved']; }, array('tank/apps/db', 'tank/apps/mv', 'tank/apps')), 'back to Default'); $pool('datasets', $apps); @@ -209,15 +222,21 @@ function same($expected, $actual, $message) { $t = 1800000000; $sends = function($pattern) use ($read) { return count(preg_grep($pattern, $read('calls.log'))); }; same(0, zfsmScheduleRun($id, $cfg, $t), 'the first copies'); +// And one below it with its own copies under usb, in place of those under backup: that copy follows too. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/data/u\tfilesystem"))); +same(true, $save('tank/data/u', $part('default'), $part('days', '05:00', '1', 0, 'usb'))['saved'], 'its own copies under usb'); +$usb = array_values(array_filter($stored(), function($job) { return $job['dataset'] === 'tank/data/u'; }))[0]['id']; +same(0, zfsmScheduleRun($usb, $cfg, $t + 60), 'their first copy'); putenv('ZFSM_TEST_PHP='.PHP_BINARY); putenv('ZFSM_TEST_LOCK_PROBE='.zfsmJobLockFile('tank/data', 'replicate')); same(array('succeeded' => array('tank/new' => 0), 'failed' => array()), renameDataset('tank/data', 'tank/new', 0, $cfg), 'a rename'); putenv('ZFSM_TEST_LOCK_PROBE'); -same(array('held', 'held'), $read('pool/probe.log'), 'its replication lock is held while it and its copy are renamed'); -same(array('backup/tank/new', 'backup/tank/new/sub'), array_values(preg_grep('~^backup/tank/(data|new)~', $datasets())), 'its copy and the one below follow it'); +same(array('held', 'held', 'held'), $read('pool/probe.log'), 'its replication lock is held while it and its copies are renamed'); +same(array(array('backup/tank/new', 'backup/tank/new/sub'), array('usb/tank/new', 'usb/tank/new/u')), array(array_values(preg_grep('~^backup/tank/(data|new)~', $datasets())), + array_values(preg_grep('~^usb/tank/(data|new)~', $datasets()))), 'its copy and the one below follow it, each under its Copy Under'); $bookmarks = function($under) use ($read) { return array_values(preg_grep('~^'.preg_quote($under, '~').'[/#]~', preg_grep('/#/', array_map(function($line) { return explode("\t", $line)[0]; }, $read('pool/marks'))))); }; -same(array('tank/new#'.zfsmReplicaBookmarkPrefix('backup/tank/new').'auto-20270115080000', 'tank/new/sub#'.zfsmReplicaBookmarkPrefix('backup/tank/new/sub').'auto-20270115080000'), - $bookmarks('tank/new'), 'their bookmarks take the copies\' new names'); +same(array('tank/new#'.zfsmReplicaBookmarkPrefix('backup/tank/new').'auto-20270115080000', 'tank/new/sub#'.zfsmReplicaBookmarkPrefix('backup/tank/new/sub').'auto-20270115080000', + 'tank/new/u#'.zfsmReplicaBookmarkPrefix('usb/tank/new/u').'auto-20270115080100'), $bookmarks('tank/new'), 'their bookmarks take the copies\' new names'); $reset(); // Its snapshots gone from the source, only the bookmarks that took the new names keep the next run incremental. $pool('marks', array_values(preg_grep('~^tank/new(/sub)?@auto-20270115080000\t~', $read('pool/marks'), PREG_GREP_INVERT))); @@ -225,6 +244,11 @@ function same($expected, $actual, $message) { $from = function($zdataset) { return preg_quote($zdataset.'#'.zfsmReplicaBookmarkPrefix('backup/'.$zdataset).'auto-20270115080000 '.$zdataset.'@', '~'); }; same(array(1, 1, 0), array($sends('~^send -P -v -L -i '.$from('tank/new').'~'), $sends('~^send -P -v -L -i '.$from('tank/new/sub').'~'), $sends('~^send -P -v -L tank/new@~')), 'sends from the moved bookmarks only the changes to the renamed copies'); +$reset(); +$pool('marks', array_values(preg_grep('~^tank/new/u@auto-20270115080100\t~', $read('pool/marks'), PREG_GREP_INVERT))); +$under = preg_quote('tank/new/u#'.zfsmReplicaBookmarkPrefix('usb/tank/new/u').'auto-20270115080100 tank/new/u@', '~'); +same(array(0, 1, 0), array(zfsmScheduleRun($usb, $cfg, $t + 3660), $sends('~^send -P -v -L -i '.$under.'~'), $sends('~^send -P -v -L tank/new/u@~')), 'and the one under usb, from its own'); +same(true, zfsmScheduleRemove($usb, $cfg)['saved'], 'its copies under usb removed, which later checks leave out'); // A run cut off part-way, then a rename: zfs finds the snapshot by guid under its new name (live, 2.4.4), so the next run carries it on. putenv('ZFSM_TEST_SEND_CUT=1'); zfsmScheduleRun($id, $cfg, $t + 7200); @@ -252,7 +276,7 @@ function same($expected, $actual, $message) { // Whether either is encrypted not read: the same, and why. renameDataset('tank/vault/in/data', 'tank/vault/data', 0, $cfg); putenv('ZFSM_TEST_CRYPT_FAIL=tank/vault/in'); -same(array('Copy backup/tank/vault/data' => 'not renamed with it: whether a dataset made at backup/tank/vault/in to hold this one would stop the copy due there could not be read. It stays under its old name, so the next copy of the renamed dataset is a full one'), +same(array('Copy backup/tank/vault/data' => 'not renamed with it: whether a dataset made at backup/tank/vault/in to hold this one would stop a copy due there could not be read. It stays under its old name, so the next copy of the renamed dataset is a full one'), renameDataset('tank/vault/data', 'tank/vault/in/data', 0, $cfg)['failed'], 'one whose encryption could not be read'); putenv('ZFSM_TEST_CRYPT_FAIL'); same(array(false, true), array(in_array('backup/tank/vault/in', $datasets(), true), in_array('backup/tank/vault/data', $datasets(), true)), 'no holder made there either'); @@ -294,6 +318,24 @@ function same($expected, $actual, $message) { array(renameDataset('tank/sealed/data', 'tank/sealed/in/data', 0, $cfg)['failed'], renameDataset('tank/vault/x', 'tank/vault/by/x', 0, $cfg)['failed'], array_map(function($zdataset) use ($datasets) { return in_array($zdataset, $datasets(), true); }, array('backup/tank/sealed/in', 'backup/tank/vault/by', 'backup/tank/sealed/data', 'backup/tank/vault/x'))), 'under an encrypted copy, and where a repeat set in Replicate is due: no holder made, and each copy stays'); +// Under one there that no run copies, as one left to Sanoid: no copy is due there, so the holder is made and the copy moves; nor +// held up where whether a run copies it could not be read. +$add(array('tank/sealed/sn', 'tank/sealed/d2', 'backup/tank/sealed/d2', 'tank/sealed/d3', 'backup/tank/sealed/d3')); +$copied('tank/sealed/d2', 73); +$copied('tank/sealed/d3', 74); +$pool('marks', array_merge($read('pool/marks'), array("tank/sealed/sn@autosnap_2027-01-15_00:00:00_daily\t9201\t900"))); +putenv('ZFSM_TEST_COVER_FAIL=tank/sealed/in'); +$unsure = renameDataset('tank/sealed/d2', 'tank/sealed/in/d2', 0, $cfg)['failed']; +putenv('ZFSM_TEST_COVER_FAIL'); +same(array(array($copy('tank/sealed/d2') => 'not renamed with it: whether a dataset made at backup/tank/sealed/in to hold this one would stop a copy due there could not be read. It stays under its old name, so the next copy of the renamed dataset is a full one'), + array(), array(true, false)), array($unsure, renameDataset('tank/sealed/d3', 'tank/sealed/sn/d3', 0, $cfg)['failed'], array(in_array('backup/tank/sealed/sn/d3', $datasets(), true), in_array('backup/tank/sealed/d3', $datasets(), true))), + 'not read: the copy stays, and said; under one no run copies, it moves'); +// Nor under one whose copies go under another Copy Under: none is due here, so the holder is made and the copy moves. +$add(array('tank/sealed/u2', 'tank/sealed/d5', 'backup/tank/sealed/d5')); +$copied('tank/sealed/d5', 75); +same(array(true, true), array($save('tank/sealed/u2', $part('default'), $part('days', '05:20', '1', 0, 'usb'))['saved'], $save('tank/sealed/d5', $part('default'), $part('days', '06:20', '1', 0, 'backup'))['saved']), 'copies under usb above, and under backup below'); +same(array(array(), true), array(renameDataset('tank/sealed/d5', 'tank/sealed/u2/d5', 0, $cfg)['failed'], in_array('backup/tank/sealed/u2/d5', $datasets(), true)), 'under one copied elsewhere, it moves'); +foreach (array_filter($stored(), function($job) { return in_array($job['dataset'], array('tank/sealed/u2', 'tank/sealed/u2/d5'), true); }) as $job) zfsmScheduleRemove($job['id'], $cfg); $add(array('tank/r', 'backup/tank/r', 'tank/q', 'backup/tank/q')); $copied('tank/q', 81); putenv('ZFSM_TEST_LIST_FAIL=backup/tank/r'); @@ -315,6 +357,21 @@ function same($expected, $actual, $message) { same(array(array('succeeded' => array(), 'failed' => array('tank/kk' => 'Nothing was renamed: the names of tank/kk/'.$kid.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name')), true), array(renameDataset('tank/k', 'tank/kk', 0, $cfg), in_array('tank/k/'.$kid, $datasets(), true)), 'a new name that leaves no room for a copy\'s bookmark'); same(array($long.'2' => 0), renameDataset($long, $long.'2', 0, $cfg)['succeeded'], 'one with no room already'); +$add(array('backup/z')); +same(array('backup/'.str_repeat('z', 200) => 0), renameDataset('backup/z', 'backup/'.str_repeat('z', 200), 0, $cfg)['succeeded'], 'one in the pool its copy would go to, which no run copies'); +// Nor one left to Sanoid, read-only or legacy-mounted, which no run copies either; one whose kind could not be read is checked. +$add(array('tank/sz', 'tank/sz/a', 'tank/sz/a/b', 'tank/ro', 'tank/lg', 'tank/un')); +$pool('marks', array_merge($read('pool/marks'), array("tank/sz@autosnap_2027-01-15_00:00:00_daily\t9301\t901", "tank/sz/a@autosnap_2027-01-15_00:00:00_daily\t9302\t902", + "tank/sz/a/b@autosnap_2027-01-15_00:00:00_daily\t9303\t903"))); +$pool('props', array_merge($read('pool/props'), array("tank/ro\treadonly\ton", "tank/lg\tmountpoint\tlegacy"))); +$longer = function($zdataset) { return $zdataset.str_repeat('w', 200); }; +$renamed = array(); +foreach (array('tank/sz', 'tank/ro', 'tank/lg') as $zdataset) $renamed += renameDataset($zdataset, $longer($zdataset), 0, $cfg)['succeeded']; +putenv('ZFSM_TEST_COVER_FAIL=tank/un'); +$unread = renameDataset('tank/un', $longer('tank/un'), 0, $cfg)['failed']; +putenv('ZFSM_TEST_COVER_FAIL'); +same(array(array($longer('tank/sz') => 0, $longer('tank/ro') => 0, $longer('tank/lg') => 0), array($longer('tank/un') => 'Nothing was renamed: the names of '.$longer('tank/un').' and its copy would be too long for the bookmark a copy keeps; pick a shorter name')), + array($renamed, $unread), 'one a run leaves out may take a long name; one not read may not'); // Its own copies go with it, so the room is that under their Copy Under. $far = 'backup/'.str_repeat('c', 20); $kin = str_repeat('n', 50); @@ -333,10 +390,10 @@ function same($expected, $actual, $message) { same(array(array('succeeded' => array(), 'failed' => array('tank/lk2' => 'Nothing was renamed: a replication of tank/lk/kid is running')), true, true), array($busy, is_resource($loose), in_array('tank/lk/kid', $datasets(), true)), 'nothing renamed while one below it copies, and its lock let go'); zfsmJobUnlock($loose); -// Docker's legacy-mounted image layers, which no run copies: not held, so not waited for. A read-only one and a legacy one a -// repeat set in Replicate copies are. -$add(array('tank/lk/layer', 'tank/lk/ro', 'tank/lk/leg')); -$pool('props', array_merge($read('pool/props'), array("tank/lk/layer\tmountpoint\tlegacy", "tank/lk/leg\tmountpoint\tlegacy", "tank/lk/ro\treadonly\ton"))); +// Docker's legacy-mounted image layers, which no run copies: one is checked, where a run by hand made its lock, but not held. A +// read-only one and a legacy one a repeat set in Replicate copies are held. +$add(array('tank/lk/layer', 'tank/lk/fresh', 'tank/lk/ro', 'tank/lk/leg')); +$pool('props', array_merge($read('pool/props'), array("tank/lk/layer\tmountpoint\tlegacy", "tank/lk/fresh\tmountpoint\tlegacy", "tank/lk/leg\tmountpoint\tlegacy", "tank/lk/ro\treadonly\ton"))); same(true, zfsmScheduleChange(function($jobs) { return $jobs + array('6f8b2c3d' => array('id' => '6f8b2c3d', 'dataset' => 'tank/lk/leg', 'dest' => 'backup/leg', 'server' => '', 'every' => 'daily', 'hour' => 3, 'day' => 0)); })['saved'], 'a repeat set in Replicate of a legacy-mounted one'); $past = array(); @@ -347,8 +404,20 @@ function same($expected, $actual, $message) { if ($past[count($past) - 1]['succeeded']) renameDataset('tank/lk3', 'tank/lk', 0, $cfg); } $running = function($zdataset) { return array('succeeded' => array(), 'failed' => array('tank/lk3' => 'Nothing was renamed: a replication of '.$zdataset.' is running')); }; -same(array(array('tank/lk3' => 0), $running('tank/lk/ro'), $running('tank/lk/leg')), array($past[0]['succeeded'], $past[1], $past[2]), - 'a legacy-mounted one no run copies is not held; a read-only one, and one a repeat copies, are'); +same(array($running('tank/lk/layer'), $running('tank/lk/ro'), $running('tank/lk/leg')), $past, 'a replication of one below it, a legacy-mounted one by hand too, is waited for'); +$fresh = zfsmJobLockFile('tank/lk/fresh', 'replicate'); +putenv('ZFSM_TEST_LOCK_PROBE='.zfsmJobLockFile('tank/lk/layer', 'replicate')); +$past = renameDataset('tank/lk', 'tank/lk3', 0, $cfg)['succeeded']; +putenv('ZFSM_TEST_LOCK_PROBE'); +$probed = $read('pool/probe.log'); +renameDataset('tank/lk3', 'tank/lk', 0, $cfg); +same(array(array('tank/lk3' => 0), 'free', false), array($past, end($probed), file_exists($fresh)), 'a legacy-mounted one no repeat copies is not held through the rename, nor its lock made'); +putenv('ZFSM_TEST_LOCK_PROBE='.zfsmJobLockFile('tank/lk/leg', 'replicate')); +$past = renameDataset('tank/lk', 'tank/lk3', 0, $cfg)['succeeded']; +putenv('ZFSM_TEST_LOCK_PROBE'); +$probed = $read('pool/probe.log'); +renameDataset('tank/lk3', 'tank/lk', 0, $cfg); +same(array(array('tank/lk3' => 0), 'held'), array($past, end($probed)), 'one a repeat copies is'); same(true, zfsmScheduleChange(function($jobs) { unset($jobs['6f8b2c3d']); return $jobs; })['saved'], 'the repeat removed'); putenv('ZFSM_TEST_TREE_FAIL=1'); $unread = renameDataset('tank/lk', 'tank/lk2', 0, $cfg)['failed']; diff --git a/tests/schedule_test.php b/tests/schedule_test.php index 1c06fffb..6d0b3410 100644 --- a/tests/schedule_test.php +++ b/tests/schedule_test.php @@ -314,7 +314,7 @@ function same($expected, $actual, $message) { $saved = file_get_contents(zfsmScheduleFile()); file_put_contents(zfsmScheduleFile(), '{"id":"'.$id.'"}'); $unreadable = zfsmScheduleFile().' could not be read as a list of schedules; fix or remove it first'; -same(array(array('tank/data' => 'Nothing was scheduled: '.$unreadable), $unreadable, '{"id":"'.$id.'"}'), array(zfsmScheduleAdd('tank/data', '', 'backup/two', '', 'days', '01:00', $every, 0, $cfg)['failed'], zfsmScheduleRemove($id)['why'], file_get_contents(zfsmScheduleFile())), 'an object is no list: nothing is added or removed'); +same(array(array('tank/data' => 'Nothing was scheduled: '.$unreadable), $unreadable, '{"id":"'.$id.'"}'), array(zfsmScheduleAdd('tank/data', '', 'backup/two', '', 'days', '01:00', $every, 0, $cfg)['failed'], zfsmScheduleRemove($id, $cfg)['why'], file_get_contents(zfsmScheduleFile())), 'an object is no list: nothing is added or removed'); same(true, strpos(zfsmSchedulesHtml($cfg), zfsmHtml($unreadable)) !== false, 'Settings says so'); $reset(); same(1, zfsmScheduleRun($id, $cfg, time()), 'and a run can\'t start'); @@ -325,12 +325,12 @@ function same($expected, $actual, $message) { // Remove takes the schedule out of the list and the cron file, and Unraid rebuilds its crontab; Settings is refused while another change is saved. $lock = zfsmJobLock(zfsmJobDir('schedule').'/settings.lock', false); -same(array('saved' => false, 'why' => 'Another change to the schedules is being saved; try again'), zfsmScheduleRemove($id), 'a change while another is being saved'); +same(array('saved' => false, 'why' => 'Another change to the schedules is being saved; try again'), zfsmScheduleRemove($id, $cfg), 'a change while another is being saved'); zfsmJobUnlock($lock); $reset(); -same(array('saved' => true, 'why' => ''), zfsmScheduleRemove($id), 'removed'); +same(array('saved' => true, 'why' => ''), zfsmScheduleRemove($id, $cfg), 'removed'); same(array(array(array('id' => 'zz')), null, array('ran')), array($stored(), $cron(), $read('cron.log')), 'the record it can\'t use stays; the cron file goes, and the crontab is rebuilt'); -same(array('No such schedule; it may have been removed already', 'No such schedule'), array(zfsmScheduleRemove($id)['why'], zfsmScheduleRemove('../x')['why']), 'removing it again, or something that is no id'); +same(array('No such schedule; it may have been removed already', 'No such schedule'), array(zfsmScheduleRemove($id, $cfg)['why'], zfsmScheduleRemove('../x', $cfg)['why']), 'removing it again, or something that is no id'); // A crontab that can't be rebuilt: the schedule is saved and its first run started, and the answer says cron runs the ones it had. $reset(); @@ -345,7 +345,7 @@ function same($expected, $actual, $message) { $weekly_id = $stored()[0]['id']; file_put_contents(zfsmScheduleStateFile($weekly_id), '{"failures":1}'); touch($work.'/cron-fail'); -same(array('saved' => true, 'why' => 'The schedules were saved, but Unraid\'s crontab could not be rebuilt (crontab: cannot write), so it runs the ones it had until the server restarts'), zfsmScheduleRemove($weekly_id), 'a removal whose crontab can\'t be rebuilt'); +same(array('saved' => true, 'why' => 'The schedules were saved, but Unraid\'s crontab could not be rebuilt (crontab: cannot write), so it runs the ones it had until the server restarts'), zfsmScheduleRemove($weekly_id, $cfg), 'a removal whose crontab can\'t be rebuilt'); unlink($work.'/cron-fail'); same(array(array(array('id' => 'zz')), false, 0), array($stored(), strpos(zfsmSchedulesHtml($cfg), 'tank/data'), zfsmScheduleFailures($weekly_id)), 'is removed from the list and Settings, with its count'); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index f94edf4e..d0dda850 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -347,10 +347,10 @@ _(Replicate Every Dataset)_: > > - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full, as does a run after a copy is gone, such as when the pool it goes into was made again. The copies are read-only and not mounted. > - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy, one whose name is too long for the bookmark its copy keeps (over about 70 characters under `backup`, fewer under a longer **Copy Under**), and, when a run can't read the snapshot names, those it couldn't read: they may be left to Sanoid or Syncoid. -> - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. If that dataset is copied later, its first copy replaces the empty one and the copies below it stay; an encrypted dataset sent raw can't replace it, nor can any copy replace an empty one that is encrypted, as one under an encrypted dataset is, and the run says what to do. When the copy of the dataset above one is missing because it failed, or the schedule that copies it has not made it yet, that one waits for the next run. +> - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. If that dataset is copied later, its first copy replaces the empty one and the copies below it stay, though not while one of them is mounted, which receiving over it would unmount; an encrypted dataset sent raw can't replace it, nor can any copy replace an empty one that is encrypted, as one under an encrypted dataset is, and the run says what to do. When the copy of the dataset above one is missing because it failed, or the schedule that copies it has not made it yet, that one waits for the next run. > - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - A dataset can have copies of its own, or None, under **Actions**, **Schedules** in the Main tab (**Replication**): it and the datasets below it follow that one instead, unless they have their own, each with its own **Copy Under**. -> - Renaming a dataset in the Main tab renames its copies under each **Copy Under** a schedule uses with it, unless a dataset already has the new name there, no schedule copies it or a dataset below it there now, what is there is neither its copy nor a holder of the copies below it, or a dataset made to hold it would stop an encrypted copy due there; it says which of those stayed, and a clash its schedules meet where they went. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves a copy no room for its bookmark. Copies under a **Copy Under** no schedule uses now stay as they are. Destroying a dataset removes what it and the datasets below it had under **Actions**, **Schedules**, and leaves its copies. +> - Renaming a dataset in the Main tab renames its copies under each **Copy Under** a schedule uses with it, unless a dataset already has the new name there, no schedule copies it or a dataset below it there now, what is there is neither its copy nor a holder of the copies below it, or a dataset made to hold it would stop an encrypted copy due there; it says which of those stayed, and a clash its schedules meet where they went. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves no room for the bookmark a copy made now keeps (named by Snapshots Pattern as it is now); a bookmark made before that no longer fits stays, and the rename says so. Copies under a **Copy Under** no schedule uses now stay as they are. Destroying a dataset removes what it and the datasets below it had under **Actions**, **Schedules**, and leaves its copies. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. > - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. A run that falls due while the one before still goes, as a first run can, is skipped and logged; once the one before has gone on for a day, it counts as a problem. > - Each run sends one notification at most: what it copied and its first problems. The first run with a problem sends it; later ones are logged in `/var/log/zfs.master.log` until a run works again, and that run's notification says so. The log has each dataset's replication. @@ -375,7 +375,7 @@ _(Schedules)_: : :zc_settings_schedules_plug: -> Snapshot Every Dataset and Replicate Every Dataset, set above, each dataset's own snapshot schedule and copies (set under **Actions**, **Schedules** in the Main tab; Remove here sets it back to Default), and the replications that run on their own. To add one of those, choose **Actions**, **Replicate** on a dataset in the Main tab and pick how often under Repeat: it runs then, and again every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. This list shows when. +> Snapshot Every Dataset and Replicate Every Dataset, set above, each dataset's own snapshot schedule and copies (set under **Actions**, **Schedules** in the Main tab; Remove here sets it back to Default, unless the schedule it then follows would clash with one as saved), and the replications that run on their own. To add one of those, choose **Actions**, **Replicate** on a dataset in the Main tab and pick how often under Repeat: it runs then, and again every hour at a minute of its own, or at the time you set on the days you tick (all seven: every day), in this server's time zone. This list shows when. > > - Each run takes a new snapshot, named by **Snapshots Pattern**, and sends the changes since the last one. If the run before it stopped part-way, it carries that on first. > - **Snapshots to keep** (set in Replicate; Remove the schedule and add it again to change it): from its second run on, once a run lands, only those it keeps of the snapshots this copy's schedules took stay, as **Snapshots To Keep** above reads it (such as `24 7d 4w 12m`), up to 50 removed a run. In a copy on this server the same snapshots go first; one held or cloned, there or here, stays on both. Snapshots made by hand, by Snapshot Every Dataset or by a schedule of another copy are never counted or removed. The one just sent counts but always stays: the next run sends the changes since it. A copy on another server keeps all of them, unless that server's **Keep Replicas** keeps fewer. Empty keeps all. diff --git a/zfs.master/backend/ZFSMAdmin.php b/zfs.master/backend/ZFSMAdmin.php index bed7d2a1..9d85b0e9 100644 --- a/zfs.master/backend/ZFSMAdmin.php +++ b/zfs.master/backend/ZFSMAdmin.php @@ -177,7 +177,9 @@ function zfsmRequireDestructiveMode($config, $subject) { break; case 'renamedataset': - zfsmReturnAnswer(renameDataset(zfsmPostString('zdataset'), zfsmPostString('newname'), zfsmPostString('force'), $zfsm_cfg), 'ZFS Dataset Rename', 'Dataset renamed successfully', 'Unable to rename dataset'); + $dataset = zfsmPostString('zdataset'); + $blocked = zfsmRequireDestructiveMode($zfsm_cfg, $dataset); + zfsmReturnAnswer($blocked ?? renameDataset($dataset, zfsmPostString('newname'), zfsmPostString('force'), $zfsm_cfg), 'ZFS Dataset Rename', 'Dataset renamed successfully', 'Unable to rename dataset'); break; case 'destroydataset': @@ -252,7 +254,7 @@ function zfsmRequireDestructiveMode($config, $subject) { case 'scheduleremove': // Saved with only its cron step failed, the list changed all the same: the answer has both. - $removed = zfsmScheduleRemove(zfsmPostString('id')); + $removed = zfsmScheduleRemove(zfsmPostString('id'), $zfsm_cfg); echo zfsmJson(($removed['why'] !== '' ? array('error' => $removed['why']) : array()) + ($removed['saved'] ? array('schedules_html' => zfsmSchedulesHtml($zfsm_cfg)) : array())); break; diff --git a/zfs.master/include/ZFSMReplicaPlan.php b/zfs.master/include/ZFSMReplicaPlan.php index d3587553..28f99b67 100644 --- a/zfs.master/include/ZFSMReplicaPlan.php +++ b/zfs.master/include/ZFSMReplicaPlan.php @@ -148,6 +148,8 @@ function zfsmReplicaPlanResume($plan, $target, $zdataset, $token) { list($held, $why) = zfsmReplicaTargetSnapshots($target, $zdest); if ($held === null) return zfsmReplicaUnread($where, $why); $holder = $held === array(); + $mounted = $holder ? zfsmReplicaHolderMounted($zdest) : ''; + if ($mounted !== '') return $where.' only holds the copies of datasets below this one, and '.$mounted; } return array_merge($plan, array('mode' => 'resume', 'snapshot' => $snapshot, 'token' => $token, 'size' => zfsmReplicateSize($info['output']), 'holder' => $holder)); } @@ -191,7 +193,8 @@ function zfsmReplicaPlanHolder($plan, $target) { $crypt = getDatasetProperty('', $plan['dest'], 'encryption'); if ($crypt === null) return $held.'whether it is encrypted could not be read'; if ($crypt !== 'off') return $held.'zfs cannot receive a copy over it, as it is encrypted'.$fix; - return array_merge($plan, array('holder' => true)); + $mounted = zfsmReplicaHolderMounted($plan['dest']); + return $mounted === '' ? array_merge($plan, array('holder' => true)) : $held.$mounted; } // What the Replicate dialog shows: the snapshots to pick from, newest first, the last destination, the trusted servers, and the plan diff --git a/zfs.master/include/ZFSMReplicaTarget.php b/zfs.master/include/ZFSMReplicaTarget.php index bf3c301a..9a37ee24 100644 --- a/zfs.master/include/ZFSMReplicaTarget.php +++ b/zfs.master/include/ZFSMReplicaTarget.php @@ -42,6 +42,17 @@ function zfsmReplicaHolderClear($zdest) { return ($cleared['code'] ?? 1) === 0 ? '' : 'The copy still carries the mark of the empty dataset it replaced: '.zfsmCommandError($cleared, 'zfs inherit failed'); } +// Why a copy received over the holder $zdest would unmount one below it, which zfs leaves so (-u; live, 2.4.4); '' when none is. +function zfsmReplicaHolderMounted($zdest) { + $listed = runProcess('zfs', array('list', '-H', '-o', 'name,mounted', '-t', 'filesystem', '-r', $zdest)); + if (($listed['code'] ?? 1) !== 0) return 'whether a copy below it is mounted could not be read: '.zfsmCommandError($listed, 'zfs list failed'); + foreach (zfsmLines($listed['output']) as $line) { + list($name, $mounted) = explode("\t", $line, 2) + array(1 => ''); + if ($mounted === 'yes') return $name.' below it is mounted, which receiving this copy over it would unmount; once it is unmounted (zfs unmount '.$name.'), the next run sends it'; + } + return ''; +} + // Whether $zdest on $target is a holder; false on a trusted server, which makes none, and when it can't be read. function zfsmReplicaHolder($target, $zdest) { if ($target['server'] !== '') return false; diff --git a/zfs.master/include/ZFSMReplicate.php b/zfs.master/include/ZFSMReplicate.php index e24125b1..d84d7284 100644 --- a/zfs.master/include/ZFSMReplicate.php +++ b/zfs.master/include/ZFSMReplicate.php @@ -199,9 +199,12 @@ function zfsmRunReplicate($zdataset, $snapshot, $zdest, $started = null, $take = } } -// A run's last step's $text and $type, after the first's when that one carried on a stopped run and warned ($first: its text, type). +// A run's last step's $text and $type, with the first's when that one carried on a stopped run and warned ($first: its text, type); +// a last step with a problem goes first, so a notification cut short still says it. function zfsmReplicateTold($first, $text, $type) { - return $first[1] === 'normal' ? array($text, $type) : array(rtrim($first[0], '.').'. '.$text, $type === 'normal' ? 'warning' : $type); + if ($first[1] === 'normal') return array($text, $type); + if ($type === 'normal') return array(rtrim($first[0], '.').'. '.$text, 'warning'); + return array(rtrim($text, '.').'. Before it: '.$first[0], $type); } // zfsmJobLock of $lock_file, waiting out another run up to $schedule's 'wait' seconds; its 'waited' is told how long. diff --git a/zfs.master/include/ZFSMSchedule.php b/zfs.master/include/ZFSMSchedule.php index 11c51ed3..b486623b 100644 --- a/zfs.master/include/ZFSMSchedule.php +++ b/zfs.master/include/ZFSMSchedule.php @@ -301,13 +301,15 @@ function zfsmScheduleSame($jobs, $job) { return ''; } -// As zfsmScheduleChange answers: removed once 'saved', even if the crontab could not be rebuilt after. -function zfsmScheduleRemove($id) { +// As zfsmScheduleChange answers: removed once 'saved', even if the crontab could not be rebuilt after; not when the schedule above +// that its datasets then follow clashes with one as saved (zfsmScheduleNewClash, Snapshots Pattern from $cfg). +function zfsmScheduleRemove($id, $cfg) { if (!zfsmScheduleId($id)) return array('saved' => false, 'why' => 'No such schedule'); - $changed = zfsmScheduleChange(function($jobs) use ($id) { + $changed = zfsmScheduleChange(function($jobs) use ($id, $cfg) { if (!isset($jobs[$id])) return 'No such schedule; it may have been removed already'; - unset($jobs[$id]); - return $jobs; + $after = array_diff_key($jobs, array($id => 0)); + $clash = zfsmScheduleNewClash($jobs, $after, $cfg); + return $clash === '' ? $after : 'Not removed: '.$clash; }); if ($changed['saved']) @unlink(zfsmScheduleStateFile($id)); return $changed; diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index 7ad3718d..f0274e4f 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -19,11 +19,15 @@ function zfsmScheduleCopiesDest($job, $zdataset) { return $job['dest'].$zdataset; } +// Whether $zdataset is in the pool $job copies into, which a run of it leaves out (zfsmScheduleCopiesNot). +function zfsmScheduleCopiesInPool($job, $zdataset) { + return explode('/', $zdataset)[0] === explode('/', $job['dest'])[0]; +} + // Why $job never copies $zdataset, listed as $row (zfsmScheduleListed), besides zfsmScheduleSkipped's reasons; $name: a new // snapshot's. '' when it copies it. function zfsmScheduleCopiesNot($job, $jobs, $zdataset, $row, $name) { - $pool = explode('/', $job['dest'])[0]; - if (explode('/', $zdataset)[0] === $pool) return $zdataset.' is in pool '.$pool.', where its copy would go'; + if (zfsmScheduleCopiesInPool($job, $zdataset)) return $zdataset.' is in pool '.explode('/', $job['dest'])[0].', where its copy would go'; if ($row[2] === null) return 'the snapshot names of '.$zdataset.' could not be read, so it may be left to Sanoid or Syncoid'; $dest = zfsmScheduleCopiesDest($job, $zdataset); if (!zfsmReplicaBookmarkFits($zdataset, zfsmReplicaTargetMark(array('server' => $job['server']), $dest), $name)) return 'the names of '.$zdataset.' and its copy are too long for the bookmark a copy keeps'; @@ -231,15 +235,18 @@ function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { // The rename's follow-up says so. if ($read === null) return ''; $snapshot = zfsmNewSnapshotName($cfg, time()); - $fits = function($zdataset, $jobs) use ($snapshot) { + // One a run leaves out (zfsmScheduleSkipped) needs no room; one whose row could not be read is checked. + $rows = zfsmScheduleListed(zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-r', $from)))[0]; + $fits = function($zdataset, $jobs, $row) use ($snapshot) { $job = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); - return $job === null || !zfsmScheduleRepeats($job['every']) + return $job === null || !zfsmScheduleRepeats($job['every']) || zfsmScheduleCopiesInPool($job, $zdataset) || ($row !== null && zfsmScheduleSkipped($zdataset, $row) !== '') || zfsmReplicaBookmarkFits($zdataset, zfsmReplicaTargetMark(array('server' => $job['server']), zfsmScheduleCopiesDest($job, $zdataset)), $snapshot); }; $after = zfsmScheduleOwnMoved($read['valid'], $from, $to); foreach ($names as $name) { $renamed = $to.substr($name, strlen($from)); - if (!$fits($renamed, $after) && $fits($name, $read['valid'])) return 'the names of '.$renamed.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'; + $row = $rows[$name] ?? null; + if (!$fits($renamed, $after, $row) && $fits($name, $read['valid'], $row)) return 'the names of '.$renamed.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'; } return ''; } @@ -255,26 +262,28 @@ function zfsmScheduleCopiesHold($zdataset) { $locks = array(); foreach (zfsmLines($listed['output']) as $line) { list($name, $guid, $mountpoint) = explode("\t", $line, 3) + array('', '', ''); - // Docker's legacy-mounted image layers, often thousands: a lock each could run out of open files. Copy Under skips them. - if ($mountpoint === 'legacy' && !isset($repeats[$name])) continue; $file = zfsmJobGuidLock($guid, 'replicate'); + // Docker's legacy-mounted image layers, often thousands, which Copy Under skips: a lock each held could run out of open files, so + // one no repeat copies is only checked, where a run made its lock. + $check = $mountpoint === 'legacy' && !isset($repeats[$name]); + if ($check && ($file === null || !file_exists($file))) continue; $lock = $file === null ? null : zfsmJobLock($file, false); - if (is_resource($lock)) { - $locks[$name] = $lock; - continue; + if (!is_resource($lock)) { + array_map('zfsmJobUnlock', $locks); + return $lock === false ? 'a replication of '.$name.' is running' : 'the replication lock of '.$name.' could not be opened'; } - array_map('zfsmJobUnlock', $locks); - return $lock === false ? 'a replication of '.$name.' is running' : 'the replication lock of '.$name.' could not be opened'; + if ($check) zfsmJobUnlock($lock); + else $locks[$name] = $lock; } return $locks; } // After zfs renamed $from to $to and its own schedules followed ($before, $after): its copy under each dataset that copies go under // takes $to's name there (zfsmScheduleCopiesMoveOne). Each with a problem, with what. -function zfsmScheduleCopiesMove($from, $to, $before, $after) { +function zfsmScheduleCopiesMove($from, $to, $before, $after, $cfg) { $stayed = array(); foreach (array_column(array_filter($before, 'zfsmScheduleCopiesUnder'), null, 'dest') as $dest => $job) { - $why = $dest === '/' ? '' : zfsmScheduleCopiesMoveOne($job, $from, $to, $after); + $why = $dest === '/' ? '' : zfsmScheduleCopiesMoveOne($job, $from, $to, $after, $cfg); if ($why !== '') $stayed[zfsmScheduleCopiesDest($job, $from)] = $why; } return $stayed; @@ -283,7 +292,7 @@ function zfsmScheduleCopiesMove($from, $to, $before, $after) { // The copy of $from under $job's dest, with its bookmarks, renamed to $to's when it is a copy of $to or a holder and a schedule in // $after still copies $to or one below it there; '' when it moved, is not there, or is not read where none copies it, else what // went wrong. -function zfsmScheduleCopiesMoveOne($job, $from, $to, $after) { +function zfsmScheduleCopiesMoveOne($job, $from, $to, $after, $cfg) { $old = zfsmScheduleCopiesDest($job, $from); $new = zfsmScheduleCopiesDest($job, $to); $missing = zfsmDatasetMissing($old); @@ -297,7 +306,7 @@ function zfsmScheduleCopiesMoveOne($job, $from, $to, $after) { if ($whose !== '') return $whose; $free = zfsmDatasetMissing($new); if ($free !== true) return $free === null ? 'not renamed with it, as whether its new name is free could not be read' : 'left where it is, as a dataset already has its new name'; - $why = zfsmScheduleCopiesRename($job['dest'], $old, $new, $after); + $why = zfsmScheduleCopiesRename($job['dest'], $old, $new, $after, $cfg); if ($why !== '') return 'not renamed with it: '.$why.'. It stays under its old name, so the next copy of the renamed dataset is a full one'; $left = implode('; ', array_filter(array(zfsmScheduleCopiesMarks($old, $new, $to), zfsmScheduleCopiesHolds($old, $new, $to)), 'strlen')); return $left === '' ? '' : 'renamed with it, but '.$left; @@ -383,14 +392,14 @@ function zfsmScheduleCopiesKept($zdataset, $dest, $jobs) { // zfs rename of copy $old to $new under $dest, after a holder is made where a level above $new is missing, as zfs rename needs its // parent (live, 2.4.4), unless that holder would stop a copy $jobs make there (zfsmScheduleCopiesBlocked). '' or why not. -function zfsmScheduleCopiesRename($dest, $old, $new, $jobs) { +function zfsmScheduleCopiesRename($dest, $old, $new, $jobs, $cfg) { $here = array(); foreach (zfsmScheduleCopiesLevels($dest, $new) as $level) { $missing = zfsmDatasetMissing($level); if ($missing === null) return 'the datasets above its new name could not be read'; if (!$missing) $here[$level] = true; } - $blocked = zfsmScheduleCopiesBlocked($dest, $new, $here, $jobs); + $blocked = zfsmScheduleCopiesBlocked($dest, $new, $here, $jobs, $cfg); if ($blocked !== '') return $blocked; $why = zfsmScheduleCopiesPlace($dest, $new, array(), $here); if ($why !== '') return $why; @@ -400,16 +409,19 @@ function zfsmScheduleCopiesRename($dest, $old, $new, $jobs) { // Of the missing levels above $zdest, a copy under $dest ($here: those there), why a holder at the first would stop a copy $jobs make // there: zfs receives no raw copy over it, nor any over an encrypted one (zfsmReplicaPlanHolder). '' when none. -function zfsmScheduleCopiesBlocked($dest, $zdest, $here, $jobs) { +function zfsmScheduleCopiesBlocked($dest, $zdest, $here, $jobs, $cfg) { $above = substr($dest, 0, -1); foreach (zfsmScheduleCopiesLevels($dest, $zdest) as $level) { if (isset($here[$level])) { $above = $level; continue; } - foreach (zfsmScheduleCopiesTo($dest, $level, $jobs) as $zdataset) { + $due = zfsmScheduleCopiesTo($dest, $level, $jobs, $cfg); + $unread = 'whether a dataset made at '.$level.' to hold this one would stop a copy due there could not be read'; + if ($due === null) return $unread; + foreach ($due as $zdataset) { $crypt = array(getDatasetProperty('', $zdataset, 'encryption'), getDatasetProperty('', $above, 'encryption')); - if (in_array(null, $crypt, true)) return 'whether a dataset made at '.$level.' to hold this one would stop the copy due there could not be read'; + if (in_array(null, $crypt, true)) return $unread; if ($crypt !== array('off', 'off')) return 'zfs could not receive the copy due at '.$level.' over the dataset made there to hold this one, as one of the two is encrypted'; } } @@ -417,9 +429,15 @@ function zfsmScheduleCopiesBlocked($dest, $zdest, $here, $jobs) { } // The datasets $jobs copy to $level under $dest, as zfsmScheduleCopiesComing reads a run's: the one of that name (above the renamed -// one, so there) when a schedule copies it under $dest, and each a repeat set in Replicate sends there. -function zfsmScheduleCopiesTo($dest, $level, $jobs) { +// one, so there) when a schedule's run takes it there (zfsmScheduleLeftOut), and each a repeat set in Replicate sends there; null +// when whether a run takes it could not be read. +function zfsmScheduleCopiesTo($dest, $level, $jobs, $cfg) { $zdataset = substr($level, strlen($dest)); - $own = (zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder')['dest'] ?? '') === $dest ? array($zdataset) : array(); - return array_merge($own, zfsmScheduleCopiesRepeatsTo($jobs)[$level] ?? array()); + $repeats = zfsmScheduleCopiesRepeatsTo($jobs)[$level] ?? array(); + $job = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); + if (($job['dest'] ?? '') !== $dest) return $repeats; + $listed = zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-d', '1', $zdataset)); + $row = zfsmScheduleListed($listed)[0][$zdataset] ?? null; + if ($row === null || $row[2] === null) return null; + return zfsmScheduleLeftOut($job, $jobs, $zdataset, $row, $listed['name']) === '' ? array_merge(array($zdataset), $repeats) : $repeats; } diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index 40ef3609..6371b560 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -247,7 +247,7 @@ function zfsmScheduleOwnFollow($answer, $from, $to, $cfg = array()) { $answer['failed']['Copies of '.$from] = 'not renamed with it, as the schedules that say where they go could not be read'; return $answer; } - foreach (zfsmScheduleCopiesMove($from, $to, $moved['before'], $moved['after']) as $copy => $why) $answer['failed']['Copy '.$copy] = $why; + foreach (zfsmScheduleCopiesMove($from, $to, $moved['before'], $moved['after'], $cfg) as $copy => $why) $answer['failed']['Copy '.$copy] = $why; return $answer; } @@ -282,11 +282,11 @@ function zfsmScheduleSnapshotCheck($jobs, $job, $cfg, $aside = array()) { } // The first clash (zfsmScheduleSnapshotCheck) $after has that $before, the same schedules by id before a change, did not, with the -// schedule it stops; '' when none. One there before stays the owner's to change. +// schedule it stops; '' when none. One there before stays the owner's to change, though another of the same schedule is new. function zfsmScheduleNewClash($before, $after, $cfg) { foreach (array_intersect_key($after, $before) as $id => $job) { $why = zfsmScheduleSnapshotCheck($after, $job, $cfg); - if ($why !== '' && zfsmScheduleSnapshotCheck($before, $before[$id], $cfg) === '') return 'for '.zfsmScheduleWhat($job).', '.$why; + if ($why !== '' && zfsmScheduleSnapshotCheck($before, $before[$id], $cfg) !== $why) return 'for '.zfsmScheduleWhat($job).', '.$why; } return ''; } From 9dc024996a2d56965b1dd93d20f35b3fbb429ffc Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:19:45 +0800 Subject: [PATCH 18/24] fix(schedules): no scheduled refill through a holder; Remove of a gone dataset's schedule; Keep help names the shared tag - A scheduled full send onto a holder where the dataset made a copy before is refused, as for a copy gone. - Remove in Settings skips the clash check for a dataset gone from an imported pool, as nothing follows it. - A rename lists the datasets below it only once a name is too long for its bookmark. - The Keep help in the Schedules dialog and Settings says Keep counts every schedule's snapshots for copies under the same Copy Under. - The mounted-copy message names the dataset without a command a space in its name would split. --- tests/schedule_copies_test.php | 15 +++++++++++++-- tests/schedule_dialog_test.php | 22 ++++++++++++++++++++-- zfs.master/ZFSMasterSettings.page | 2 +- zfs.master/backend/ZFSMOperations.php | 7 ++++++- zfs.master/frontend/ZFSMImpact.js | 2 +- zfs.master/include/ZFSMReplicaPlan.php | 8 +++++--- zfs.master/include/ZFSMReplicaTarget.php | 2 +- zfs.master/include/ZFSMSchedule.php | 4 +++- zfs.master/include/ZFSMScheduleCopies.php | 18 +++++++++++------- 9 files changed, 61 insertions(+), 19 deletions(-) diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index ef1dd055..33e303c7 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -212,10 +212,21 @@ function same($expected, $actual, $message) { $pool('props', array_merge($read('pool/props'), array("backup/tank/mnt\tzfs.master:holder\ton", "backup/tank/mnt/kid\tmounted\tyes"))); $plan = function() { return zfsmReplicatePlan('tank/mnt', 'a', 'backup/tank/mnt', array('estimate' => false)); }; $over = 'backup/tank/mnt only holds the copies of datasets below this one, and '; -same(array('can_proceed' => false, 'error' => $over.'backup/tank/mnt/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted (zfs unmount backup/tank/mnt/kid), the next run sends it'), $plan(), 'one with a copy below it mounted'); +same(array('can_proceed' => false, 'error' => $over.'backup/tank/mnt/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted, the next run sends it'), $plan(), 'one with a copy below it mounted'); putenv('ZFSM_TEST_MOUNTED_FAIL=1'); same(array('can_proceed' => false, 'error' => $over.'whether a copy below it is mounted could not be read: cannot open \'backup/tank/mnt\': Input/output error'), $plan(), 'one where that could not be read'); putenv('ZFSM_TEST_MOUNTED_FAIL'); +// A schedule's full send onto a holder made where its copy was, as a rename below it makes one, is refused as for a copy gone; by +// hand, or where the dataset made no copy before, it goes. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/rf\tfilesystem", "tank/rf2\tfilesystem", "backup/tank/rf\tfilesystem", "backup/tank/rf2\tfilesystem"))); +$pool('marks', array_merge($read('pool/marks'), array("tank/rf@a\t4848\t9007", 'tank/rf#'.zfsmReplicaBookmarkPrefix('backup/tank/rf')."a\t4848\t9007", "tank/rf2@a\t4949\t9008"))); +$pool('props', array_merge($read('pool/props'), array("backup/tank/rf\tzfs.master:holder\ton", "backup/tank/rf2\tzfs.master:holder\ton"))); +$refill = function($zdataset, $scheduled) { + $plan = zfsmReplicatePlan($zdataset, 'a', 'backup/'.$zdataset, array('estimate' => false, 'scheduled' => $scheduled)); + return $plan['can_proceed'] ? $plan['holder'] : $plan['error']; +}; +same(array('backup/tank/rf only holds the copies of datasets below this one, though this dataset made a copy there before; a scheduled run sends only changes, so replicate it once by hand to make a new copy', true, true), + array($refill('tank/rf', true), $refill('tank/rf', false), $refill('tank/rf2', true)), 'a scheduled one where it made a copy is refused; by hand, or where it made none, it goes'); // A first copy onto a holder that stops part-way carries on only with -F again (OpenZFS 2.4.4); the mark goes once it lands. $pool('datasets', array_merge($read('pool/datasets'), array("tank/res\tfilesystem", "backup/tank/res\tfilesystem"))); @@ -228,7 +239,7 @@ function same($expected, $actual, $message) { same(array(true, 'resume'), array(zfsmReplicaHolder(array('server' => ''), 'backup/tank/res'), zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false))['mode']), 'still a holder, part-received'); $pool('datasets', array_merge($read('pool/datasets'), array("backup/tank/res/kid\tfilesystem"))); $pool('props', array_merge($read('pool/props'), array("backup/tank/res/kid\tmounted\tyes"))); -same(array('can_proceed' => false, 'error' => 'backup/tank/res only holds the copies of datasets below this one, and backup/tank/res/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted (zfs unmount backup/tank/res/kid), the next run sends it'), +same(array('can_proceed' => false, 'error' => 'backup/tank/res only holds the copies of datasets below this one, and backup/tank/res/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted, the next run sends it'), zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false)), 'nor carried on over a copy below it mounted'); $pool('props', array_values(preg_grep("~^backup/tank/res/kid\tmounted\t~", $read('pool/props'), PREG_GREP_INVERT))); putenv('ZFSM_TEST_MARKS_FAIL=backup/tank/res'); diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index c6053ad0..6083a749 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -168,6 +168,20 @@ function same($expected, $actual, $message) { preg_match('/^moved with it, but for (the copies of tank\/apps\/mv and below|the snapshot schedule of tank\/apps), Snapshots Pattern would give the run at .* the same snapshot name as .*; add the time to Snapshots Pattern$/', $moved['failed']['Schedules of tank/apps/mv'] ?? ''), array_values(preg_grep('~^tank/(apps/)?mv$~', array_column($stored(), 'dataset')))), 'a rename into a clash says it, and its schedule moves'); +// Of a dataset gone, as one destroyed outside the plugin, Remove goes though Default would clash, as no dataset follows it; not of one in +// a pool not imported, which may be there. Each a copy of the schedules of tank/apps and tank/apps/db. +$clone = function($root) { return zfsmScheduleChange(function($jobs) use ($root) { + foreach ($jobs as $job) if (in_array($job['dataset'], array('tank/apps', 'tank/apps/db'), true)) { + $job['id'] = substr(hash('sha256', $root.$job['id']), 0, 8); + $job['dataset'] = $root.substr($job['dataset'], strlen('tank/apps')); + $jobs[$job['id']] = $job; + } + return $jobs; +})['saved']; }; +$own_snap = function($zdataset) use ($stored) { return array_values(array_filter($stored(), function($job) use ($zdataset) { return $job['dataset'] === $zdataset && zfsmScheduleSnapshotsOnly($job); }))[0]['id']; }; +same(array(true, true, true, 1), array($clone('tank/gone'), $clone('usb2/apps'), zfsmScheduleRemove($own_snap('tank/gone/db'), array('snap_pattern' => 'Ymd') + $cfg)['saved'], + preg_match('/^Not removed: for /', zfsmScheduleRemove($own_snap('usb2/apps/db'), array('snap_pattern' => 'Ymd') + $cfg)['why'])), 'a gone one goes; one in a pool not imported stays'); +zfsmScheduleChange(function($jobs) { return array_filter($jobs, function($job) { return !zfsmUnderAny($job['dataset'], array('tank/gone', 'usb2/apps')); }); }); // Remove in Settings sets a dataset's own back to Default too: checked as that is. $own = array_values(array_filter($stored(), function($job) { return $job['dataset'] === 'tank/apps/db' && zfsmScheduleSnapshotsOnly($job); }))[0]['id']; $removed = array(zfsmScheduleRemove($own, array('snap_pattern' => 'Ymd') + $cfg)); @@ -318,8 +332,8 @@ function same($expected, $actual, $message) { array(renameDataset('tank/sealed/data', 'tank/sealed/in/data', 0, $cfg)['failed'], renameDataset('tank/vault/x', 'tank/vault/by/x', 0, $cfg)['failed'], array_map(function($zdataset) use ($datasets) { return in_array($zdataset, $datasets(), true); }, array('backup/tank/sealed/in', 'backup/tank/vault/by', 'backup/tank/sealed/data', 'backup/tank/vault/x'))), 'under an encrypted copy, and where a repeat set in Replicate is due: no holder made, and each copy stays'); -// Under one there that no run copies, as one left to Sanoid: no copy is due there, so the holder is made and the copy moves; nor -// held up where whether a run copies it could not be read. +// Under one there that no run copies, as one left to Sanoid: no copy is due there, so the holder is made and the copy moves; where +// whether a run copies it could not be read, it stays. $add(array('tank/sealed/sn', 'tank/sealed/d2', 'backup/tank/sealed/d2', 'tank/sealed/d3', 'backup/tank/sealed/d3')); $copied('tank/sealed/d2', 73); $copied('tank/sealed/d3', 74); @@ -439,12 +453,16 @@ function same($expected, $actual, $message) { $add(array('tank/p', 'tank/p/kid', 'backup/tank/p', 'backup/tank/p/kid')); $pool('props', array_merge($read('pool/props'), array("backup/tank/p\tzfs.master:holder\ton"))); same(array(true, true), array($save('tank/p', $part('default'), $part('none'))['saved'], $save('tank/p/kid', $part('default'), $part('hourly', '', '', 0, 'backup'))['saved']), 'None above, its own below'); +$logged = count($read('calls.log')); same(array('succeeded' => array('tank/p2' => 0), 'failed' => array()), renameDataset('tank/p', 'tank/p2', 0, $cfg), 'a rename of the one above'); +same(array(), preg_grep('/^list -H -o name,readonly,mountpoint -t filesystem,volume -r /', array_slice($read('calls.log'), $logged)), 'whose names fit: the datasets below it are not listed'); same(array('backup/tank/p2', 'backup/tank/p2/kid'), array_values(preg_grep('~^backup/tank/p~', $datasets())), 'its copy and the one below follow it'); // Its schedules not moved, as another change holds them: its copies follow as they would have, ready for its schedules set again. $lock = zfsmJobLock(zfsmJobDir('schedule').'/settings.lock', false); +$kept = $stored(); same(array('Schedules of tank/p2'), array_keys(renameDataset('tank/p2', 'tank/p3', 0, $cfg)['failed']), 'only the schedules are said'); zfsmJobUnlock($lock); +same($kept, $stored(), 'its schedules stay as they were'); same(array('backup/tank/p3', 'backup/tank/p3/kid'), array_values(preg_grep('~^backup/tank/p~', $datasets())), 'its copies follow it'); // One with no copy there: nothing to say. same(array('succeeded' => array('tank/r3' => 0), 'failed' => array()), renameDataset('tank/r2', 'tank/r3', 0, $cfg), 'one with no copy'); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index d0dda850..cd5dc54b 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -348,7 +348,7 @@ _(Replicate Every Dataset)_: > - Each run takes a new snapshot of each dataset, named by **Snapshots Pattern**, and sends the changes since the last one, as a copy's schedule does (see **Schedules** below): the first run sends each dataset in full, as does a run after a copy is gone, such as when the pool it goes into was made again. The copies are read-only and not mounted. > - It leaves out the datasets in the pool it copies into, and the ones Snapshot Every Dataset leaves out: read-only datasets, such as copies received here, those mounted `legacy`, and those with Sanoid or Syncoid snapshots. It also leaves out a dataset whose own copy schedule (**Actions**, **Replicate** in the Main tab) makes that same copy, one whose name is too long for the bookmark its copy keeps (over about 70 characters under `backup`, fewer under a longer **Copy Under**), and, when a run can't read the snapshot names, those it couldn't read: they may be left to Sanoid or Syncoid. > - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. If that dataset is copied later, its first copy replaces the empty one and the copies below it stay, though not while one of them is mounted, which receiving over it would unmount; an encrypted dataset sent raw can't replace it, nor can any copy replace an empty one that is encrypted, as one under an encrypted dataset is, and the run says what to do. When the copy of the dataset above one is missing because it failed, or the schedule that copies it has not made it yet, that one waits for the next run. -> - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only the snapshots this schedule took that it keeps stay, on the dataset and in its copy. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. +> - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only those it keeps of the snapshots taken for copies under its **Copy Under** stay, on the dataset and in its copy, whichever schedule took them: a dataset's own copies under the same one share them. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - A dataset can have copies of its own, or None, under **Actions**, **Schedules** in the Main tab (**Replication**): it and the datasets below it follow that one instead, unless they have their own, each with its own **Copy Under**. > - Renaming a dataset in the Main tab renames its copies under each **Copy Under** a schedule uses with it, unless a dataset already has the new name there, no schedule copies it or a dataset below it there now, what is there is neither its copy nor a holder of the copies below it, or a dataset made to hold it would stop an encrypted copy due there; it says which of those stayed, and a clash its schedules meet where they went. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves no room for the bookmark a copy made now keeps (named by Snapshots Pattern as it is now); a bookmark made before that no longer fits stays, and the rename says so. Copies under a **Copy Under** no schedule uses now stay as they are. Destroying a dataset removes what it and the datasets below it had under **Actions**, **Schedules**, and leaves its copies. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. diff --git a/zfs.master/backend/ZFSMOperations.php b/zfs.master/backend/ZFSMOperations.php index 42008a6d..f8170599 100644 --- a/zfs.master/backend/ZFSMOperations.php +++ b/zfs.master/backend/ZFSMOperations.php @@ -53,6 +53,11 @@ function zfsmDatasetMissing($name) { return zfsmDatasetMissingSaid($result['output'] ?? '') ? true : null; } +// True when ZFS says it does not exist in a pool imported now; one in a pool not imported, or not read, may be there. +function zfsmDatasetGone($name) { + return zfsmDatasetMissing($name) === true && zfsmDatasetMissing(strtok($name, '/')) === false; +} + // Whether a failed zfs command's output says its dataset does not exist. Only the closing diagnostic counts: a name may itself contain the words. function zfsmDatasetMissingSaid($output) { return preg_match('/: dataset does not exist\s*\z/', (string)$output) === 1; @@ -626,7 +631,7 @@ function zfsmSetShareOwner($pool, $names) { function renameDataset($zdataset, $new_name, $force, $cfg) { if (!zfsmDatasetExists($zdataset) || !zfsmValidDatasetName($new_name)) return zfsmFailure($new_name, 'Invalid source or destination dataset'); - // Held until its copies follow it, so no run copies it under its new name first. + // Held until its copies follow it, so no run sends it under its new name first. $locks = zfsmScheduleCopiesHold($zdataset); if (is_string($locks)) return zfsmFailure($new_name, 'Nothing was renamed: '.$locks); try { diff --git a/zfs.master/frontend/ZFSMImpact.js b/zfs.master/frontend/ZFSMImpact.js index a79bc98c..ecc10c95 100644 --- a/zfs.master/frontend/ZFSMImpact.js +++ b/zfs.master/frontend/ZFSMImpact.js @@ -160,7 +160,7 @@ function zfsmScheduleDialogHtml(preview) { +say('Datasets below this one follow its choice, unless they have their own.') +say('Replication copies this dataset and those below into Copy Under, a dataset on this server in another pool, each by its full name: tank/data goes to backup/tank/data. Each copy is read-only and unmounted; a run sends only the changes since its newest snapshot, or the whole dataset when it has no copy there yet.') +say('Read-only datasets, such as copies received here, those mounted legacy, such as Docker\'s image layers, and those with Sanoid or Syncoid snapshots (autosnap_, syncoid_; Settings says when Snapshots Prefix counts instead) are never taken or copied. Schedules saves no repeat of either part on such a dataset itself, but a dataset below it that a run can take may have its own.') - +say('Snapshots to keep: for Snapshots, after each run only the scheduled snapshots it keeps stay on each dataset, whichever snapshot schedule took them; for Replication, only those it keeps of the snapshots it took for its copies stay, on each dataset and in its copy. A run removes up to 50 from each, and one held or cloned stays. '+zfsmKeepHelp()+' Snapshots made by hand are never removed.') + +say('Snapshots to keep: for Snapshots, after each run only the scheduled snapshots it keeps stay on each dataset, whichever snapshot schedule took them; for Replication, only those it keeps of the snapshots taken for copies under the same Copy Under stay, on each dataset and in its copy, whichever schedule took them, such as Replicate Every Dataset. A run removes up to 50 from each, and one held or cloned stays. '+zfsmKeepHelp()+' Snapshots made by hand are never removed.') +say('Settings lists every schedule.')+''; } diff --git a/zfs.master/include/ZFSMReplicaPlan.php b/zfs.master/include/ZFSMReplicaPlan.php index 28f99b67..966080ad 100644 --- a/zfs.master/include/ZFSMReplicaPlan.php +++ b/zfs.master/include/ZFSMReplicaPlan.php @@ -123,10 +123,12 @@ function zfsmReplicatePlan($zdataset, $snapshot, $zdest, $with = array()) { if (!zfsmReplicaBookmarkFits($zdataset, $key, $from !== null ? $snapshot : ($with['new_name'] ?? ''))) return $refuse(ZFSM_REPLICA_TOO_LONG); if (!$missing) $plan = zfsmReplicaPlanOnto($plan, $target, $zdataset, $marks, $snap); if (is_string($plan)) return $refuse($plan); - // A bookmark for this copy means a run made it before: a schedule's full send there would refill what someone removed. - if ($missing && ($with['scheduled'] ?? false)) { + // A bookmark for this copy means a run made it before: a schedule's full send there, or onto a holder made there since, would + // refill what someone removed. + if (($missing || $plan['holder']) && ($with['scheduled'] ?? false)) { $prefix = $zdataset.'#'.zfsmReplicaBookmarkPrefix($key); - foreach ($marks as $mark) if (strncmp($mark['name'], $prefix, strlen($prefix)) === 0) return $refuse($where.' is gone, though this dataset made a copy there before; a scheduled run sends only changes, so replicate it once by hand to make a new copy'); + $now = $missing ? ' is gone' : ' only holds the copies of datasets below this one'; + foreach ($marks as $mark) if (strncmp($mark['name'], $prefix, strlen($prefix)) === 0) return $refuse($where.$now.', though this dataset made a copy there before; a scheduled run sends only changes, so replicate it once by hand to make a new copy'); } return array_merge($plan, array('size' => $from === null || ($with['estimate'] ?? true) === false ? null : zfsmReplicateEstimate($plan))); } diff --git a/zfs.master/include/ZFSMReplicaTarget.php b/zfs.master/include/ZFSMReplicaTarget.php index 9a37ee24..1ce1b64c 100644 --- a/zfs.master/include/ZFSMReplicaTarget.php +++ b/zfs.master/include/ZFSMReplicaTarget.php @@ -48,7 +48,7 @@ function zfsmReplicaHolderMounted($zdest) { if (($listed['code'] ?? 1) !== 0) return 'whether a copy below it is mounted could not be read: '.zfsmCommandError($listed, 'zfs list failed'); foreach (zfsmLines($listed['output']) as $line) { list($name, $mounted) = explode("\t", $line, 2) + array(1 => ''); - if ($mounted === 'yes') return $name.' below it is mounted, which receiving this copy over it would unmount; once it is unmounted (zfs unmount '.$name.'), the next run sends it'; + if ($mounted === 'yes') return $name.' below it is mounted, which receiving this copy over it would unmount; once it is unmounted, the next run sends it'; } return ''; } diff --git a/zfs.master/include/ZFSMSchedule.php b/zfs.master/include/ZFSMSchedule.php index b486623b..c7dafbd8 100644 --- a/zfs.master/include/ZFSMSchedule.php +++ b/zfs.master/include/ZFSMSchedule.php @@ -302,13 +302,15 @@ function zfsmScheduleSame($jobs, $job) { } // As zfsmScheduleChange answers: removed once 'saved', even if the crontab could not be rebuilt after; not when the schedule above -// that its datasets then follow clashes with one as saved (zfsmScheduleNewClash, Snapshots Pattern from $cfg). +// that its datasets then follow clashes with one as saved (zfsmScheduleNewClash, Snapshots Pattern from $cfg), unless its dataset is +// gone (zfsmDatasetGone), so none follows it. function zfsmScheduleRemove($id, $cfg) { if (!zfsmScheduleId($id)) return array('saved' => false, 'why' => 'No such schedule'); $changed = zfsmScheduleChange(function($jobs) use ($id, $cfg) { if (!isset($jobs[$id])) return 'No such schedule; it may have been removed already'; $after = array_diff_key($jobs, array($id => 0)); $clash = zfsmScheduleNewClash($jobs, $after, $cfg); + if ($clash !== '' && $jobs[$id]['dataset'] !== '' && zfsmDatasetGone($jobs[$id]['dataset'])) $clash = ''; return $clash === '' ? $after : 'Not removed: '.$clash; }); if ($changed['saved']) @unlink(zfsmScheduleStateFile($id)); diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index f0274e4f..c8c291cb 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -235,18 +235,22 @@ function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { // The rename's follow-up says so. if ($read === null) return ''; $snapshot = zfsmNewSnapshotName($cfg, time()); - // One a run leaves out (zfsmScheduleSkipped) needs no room; one whose row could not be read is checked. - $rows = zfsmScheduleListed(zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-r', $from)))[0]; - $fits = function($zdataset, $jobs, $row) use ($snapshot) { + // One a run leaves out (zfsmScheduleSkipped) needs no room; one whose row could not be read is checked. Read once a name is too long. + $rows = null; + $skipped = function($zdataset, $name) use (&$rows, $cfg, $from) { + if ($rows === null) $rows = zfsmScheduleListed(zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-r', $from)))[0]; + return isset($rows[$name]) && zfsmScheduleSkipped($zdataset, $rows[$name]) !== ''; + }; + $fits = function($zdataset, $jobs, $name) use ($snapshot, $skipped) { $job = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); - return $job === null || !zfsmScheduleRepeats($job['every']) || zfsmScheduleCopiesInPool($job, $zdataset) || ($row !== null && zfsmScheduleSkipped($zdataset, $row) !== '') - || zfsmReplicaBookmarkFits($zdataset, zfsmReplicaTargetMark(array('server' => $job['server']), zfsmScheduleCopiesDest($job, $zdataset)), $snapshot); + return $job === null || !zfsmScheduleRepeats($job['every']) || zfsmScheduleCopiesInPool($job, $zdataset) + || zfsmReplicaBookmarkFits($zdataset, zfsmReplicaTargetMark(array('server' => $job['server']), zfsmScheduleCopiesDest($job, $zdataset)), $snapshot) + || $skipped($zdataset, $name); }; $after = zfsmScheduleOwnMoved($read['valid'], $from, $to); foreach ($names as $name) { $renamed = $to.substr($name, strlen($from)); - $row = $rows[$name] ?? null; - if (!$fits($renamed, $after, $row) && $fits($name, $read['valid'], $row)) return 'the names of '.$renamed.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'; + if (!$fits($renamed, $after, $name) && $fits($name, $read['valid'], $name)) return 'the names of '.$renamed.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'; } return ''; } From c849e0358620c2e85779e0ae540233c48db32cce Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:43:51 +0800 Subject: [PATCH 19/24] fix(schedules): a rename that lands but fails to mount is followed; a copy another schedule sends into stays; no holder for a dataset gone mid-run - zfs 2.0 and 2.1 exit 1 when a rename lands but its remount fails: its schedules and copies now follow, and the error stays - A rename leaves a copy in place while another schedule's Copy Under or destination is inside it, and names that schedule - A run makes no holder for a dataset renamed or destroyed since it listed it - Clash checks leave out the dataset that a repeat set in Replicate copies to the same place, as runs do - The holder refusal now warns that a destroyed copy loses its snapshots before giving the destroy step; the mounted-copy message no longer promises a send - Copy Under is sent as typed; the help says a repeat saved before on a dataset that runs leave out stays - Fixture header is two lines; version notes are gone from comments --- tests/fixtures/bin/zfs | 17 ++++++------- tests/page_dialog_test.cjs | 6 ++--- tests/replicate_test.php | 2 +- tests/schedule_copies_test.php | 28 +++++++++++++++------ tests/schedule_dialog_test.php | 24 +++++++++++++++++- zfs.master/ZFSMasterSettings.page | 2 +- zfs.master/frontend/ZFSMImpact.js | 4 +-- zfs.master/include/ZFSMReplicaPlan.php | 8 +++--- zfs.master/include/ZFSMReplicaTarget.php | 4 +-- zfs.master/include/ZFSMSchedule.php | 2 ++ zfs.master/include/ZFSMScheduleCopies.php | 27 +++++++++++++++++--- zfs.master/include/ZFSMScheduleSnapshot.php | 9 +++---- 12 files changed, 95 insertions(+), 38 deletions(-) diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index 064a50f6..fe3a527c 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -15,11 +15,8 @@ for arg in "$@"; do last_arg="$arg"; done # Every call, for a test that counts how many reads a page makes. if [ -n "${ZFSM_TEST_CALL_LOG:-}" ]; then printf '%s\n' "$*" >> "$ZFSM_TEST_CALL_LOG"; fi -# Replication (include/ZFSMReplicate.php) against a small pool kept in ZFSM_TEST_REPL_DIR: datasets ("nametype"), -# marks (snapshots and bookmarks, "nameguidcreatetxg", oldest first), props ("namepropertyvalue", a -# snapshot's clones and user properties too) and holds ("snapshottag"). destroy refuses a held or cloned snapshot, as ZFS does. -# send writes a stream naming its snapshot and base; receive checks it as ZFS does and keeps a token for a cut-off stream, and -# the snapshot guid that token carries in tokens ("tokenguid"). +# Replication (include/ZFSMReplicate.php) against the pool in ZFSM_TEST_REPL_DIR: datasets, marks (oldest first), props, holds and +# tokens ("tokenguid"), tab-separated. destroy refuses a held or cloned snapshot; receive makes only the checks below. if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n "$@"; }; then # a dry run: answered below, pool unchanged R=$ZFSM_TEST_REPL_DIR touch "$R/datasets" "$R/marks" "$R/props" "$R/holds" @@ -173,7 +170,7 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n snap=$last_arg if [ -n "$token" ]; then toname=${token#tok:}; snap=$toname; base=RESUME - # As zfs: by its guid, in its pool, under the name it has now (live, 2.4.4: after a rename too); by name for a token a test wrote. + # As zfs: by its guid, in its pool, under the name it has now (after a rename too); by name for a token a test wrote. g=$(awk -F'\t' -v t="$token" '$1 == t { g = $2 } END { print g }' "$R/tokens" 2>/dev/null) [ -z "$g" ] || snap=$(awk -F'\t' -v g="$g" -v n="$toname" -v p="${toname%%[/@]*}" '$2 == g && $1 == n { e = $1 } $2 == g && index($1, "@") && (index($1, p "/") == 1 || index($1, p "@") == 1) && f == "" { f = $1 } END { print e != "" ? e : f }' "$R/marks") [ -n "$snap" ] || { printf "cannot resume send: '%s' used in the initial send no longer exists\n" "$toname" >&2; exit 255; } @@ -206,11 +203,11 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n [ "${ZFSM_TEST_RECV_FAIL:-0}" = 1 ] && { printf 'cannot receive new filesystem stream: out of space\n' >&2; exit 1; } snap=$(printf '%s' "$header" | cut -f2); base=$(printf '%s' "$header" | cut -f3); guid=$(printf '%s' "$header" | cut -f4) [ "${ZFSM_TEST_RECV_FAIL:-0}" = changes ] && [ "$base" != - ] && [ "$base" != RESUME ] && { printf 'cannot receive incremental stream: out of space\n' >&2; exit 1; } - # A full stream onto one that is there needs -F; one cut off there carries on only with -F too (2.4.4, live). + # A full stream onto one that is there needs -F; one cut off there carries on only with -F too. if [ "$base" = - ] && exists "$dest" && [ "$(prop "$dest" receive_resume_token -)" = - ] && [ "$force" = 0 ]; then printf "cannot receive new filesystem stream: destination '%s' exists\n" "$dest" >&2; exit 1; fi if [ "$base" = RESUME ] && [ "$(prop "$dest" zfsm_test_onto -)" = 1 ] && [ "$force" = 0 ]; then printf "cannot receive resume stream: Resuming recv on existing destination '%s'\nmust specify -F to overwrite it\n" "$dest" >&2; exit 1; fi onto=0; [ "$base" = - ] && exists "$dest" && onto=1 - # Nor onto an encrypted one (live, 2.4.4). + # Nor onto an encrypted one. if [ "$onto" = 1 ] && [ "$(prop "$dest" encryption off)" != off ]; then printf 'cannot receive new filesystem stream: zfs receive -F cannot be used to destroy an encrypted filesystem or overwrite an unencrypted one with an encrypted one\n' >&2; exit 1; fi # zfs makes a new copy only inside a dataset that exists. exists "$dest" || exists "${dest%/*}" || { printf "cannot receive new filesystem stream: parent of '%s' does not exist\n" "$dest" >&2; exit 1; } @@ -238,7 +235,8 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n awk -F'\t' -v n="$last_arg" -v p="$2" '!($1 == n && $2 == p)' "$R/props" > "$R/props.new" && mv "$R/props.new" "$R/props" exit 0 ;; rename) - # Moves it and all below it, with their snapshots, bookmarks, properties and holds; ZFSM_TEST_RENAME_FAIL names one that is busy. + # Moves it and all below it, with their snapshots, bookmarks, properties and holds; ZFSM_TEST_RENAME_FAIL names one that is busy, + # ZFSM_TEST_RENAME_UNMOUNTED one that moves, then fails to mount there. # ZFSM_TEST_LOCK_PROBE: a lock file whose state at each rename goes to probe.log, held or free, as ZFSM_TEST_PHP finds it. [ -z "${ZFSM_TEST_LOCK_PROBE:-}" ] || "$ZFSM_TEST_PHP" -r 'echo flock(fopen(getenv("ZFSM_TEST_LOCK_PROBE"), "c"), LOCK_EX | LOCK_NB) ? "free\n" : "held\n";' >> "$R/probe.log" eval "from=\${$(($# - 1))}" @@ -250,6 +248,7 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n for f in datasets marks props holds; do awk -F'\t' -v OFS='\t' -v a="$from" -v b="$last_arg" '{ n = length(a); c = substr($1, n + 1, 1); if (substr($1, 1, n) == a && (c == "" || c == "/" || c == "@" || c == "#")) $1 = b substr($1, n + 1); print }' "$R/$f" > "$R/$f.new" && mv "$R/$f.new" "$R/$f" done + [ "${ZFSM_TEST_RENAME_UNMOUNTED:-}" = "$from" ] && { printf "cannot mount '/mnt/%s': mountpoint is not a directory\n" "$last_arg" >&2; exit 1; } exit 0 ;; bookmark) guid=$(mark "$2" 2); txg=$(mark "$2" 3) diff --git a/tests/page_dialog_test.cjs b/tests/page_dialog_test.cjs index 91ac2e3e..d9e5a891 100644 --- a/tests/page_dialog_test.cjs +++ b/tests/page_dialog_test.cjs @@ -489,11 +489,11 @@ async function confirm() { window.document.querySelectorAll('.zfsm-copies-day').forEach(box => { box.checked = box.value === '6'; }); copy('time').value = '02:15'; copy('keep').value = ' 7d '; - copy('under').value = ' backup '; + copy('under').value = 'backup/Cold Storage '; scheduleAnswer = JSON.stringify({said: 'Saved: tank/data', schedules_html: 'x'}); await confirm(); - assert.deepEqual(posts.at(-1), {cmd: 'scheduledataset', zdataset: 'tank/data', every: 'days', time: '13:30', days: '1,3', keep: '2', copy_every: 'days', copy_time: '02:15', copy_days: '6', copy_keep: '7d', under: 'backup'}, - 'Save sends both parts as on screen'); + assert.deepEqual(posts.at(-1), {cmd: 'scheduledataset', zdataset: 'tank/data', every: 'days', time: '13:30', days: '1,3', keep: '2', copy_every: 'days', copy_time: '02:15', copy_days: '6', copy_keep: '7d', under: 'backup/Cold Storage '}, + 'Save sends both parts as on screen, Copy Under as typed: a name may end in a space'); const text = () => window.Swal2.getHtmlContainer().textContent; const result = () => [window.Swal2.getTitle().textContent, window.Swal2.getIcon().classList.contains('swal2-error'), text()]; assert.deepEqual(result(), ['Schedules Result', false, 'Saved: tank/data'], 'and shows what was saved, as text'); diff --git a/tests/replicate_test.php b/tests/replicate_test.php index 948f1c7d..998b13f0 100644 --- a/tests/replicate_test.php +++ b/tests/replicate_test.php @@ -123,7 +123,7 @@ function same($expected, $actual, $message) { same(array('normal: Replication of tank/data@b to backup/data finished (carried on from where it stopped).'), $read('notify-detail.log'), 'the notification says it carried on'); file_put_contents($work.'/pool/props', "backup/data\treceive_resume_token\ttok:tank/vol@v\n", FILE_APPEND); same('backup/data holds a part-received replication of tank/vol@v, not of this dataset', $refuse('b', 'backup/data'), 'another dataset\'s token is not resumed'); -// zfs send -P -v -n -t as it prints live (2.4.4) after the source and the copy were renamed, values made up: toname still says src@b. +// zfs send -P -v -n -t as it prints after the source and the copy were renamed, values made up: toname still says src@b. $dry = file_get_contents(__DIR__.'/fixtures/zfs_send_resume_renamed.txt'); same(array('tank/new@b', 12345678), array(zfsmReplicaResumeSnapshot($dry), zfsmReplicateSize($dry)), 'a dry run in the live shape, after a rename, gives the snapshot\'s name now and its size'); diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 33e303c7..703af3ff 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -91,6 +91,12 @@ function same($expected, $actual, $message) { 'a nearer one of its kind, None too, takes the datasets below it from one above'); same('the copies of tank/data/sub and below (every hour at :30) starts then too; pick another time', zfsmScheduleSame(array($mine['id'] => $mine), $tank), 'one above is checked against one below it before it is in the list'); +// A repeat set in Replicate to the copy a run of copies would make of its dataset takes that one out of the run: they share none. +$repeat = array('id' => '1a2b3c4d', 'dataset' => 'tank/data/sub', 'dest' => 'backup/tank/data/sub', 'server' => '', 'every' => 'hourly') + zfsmScheduleAt('hourly', '', '') + array('keep' => 0); +same(array(false, false, false, true, true), + array(zfsmScheduleShare($every, $repeat, array()), zfsmScheduleShare($repeat, $every, array()), zfsmScheduleShare(array('dataset' => 'tank/data/sub') + $every, $repeat, array()), + zfsmScheduleShare($every, array('dest' => 'usb/tank/data/sub') + $repeat, array()), zfsmScheduleShare(array('dest' => 'usb/') + $every, $repeat, array())), + 'a repeat set in Replicate to the copy a run of copies makes there shares no dataset with it; to another, it does'); // Covered: each dataset but those left out, by pool; not one whose own copy's schedule makes that copy. $listed = function() use ($cfg) { return zfsmScheduleCoverRead($cfg, 10); }; @@ -194,25 +200,25 @@ function same($expected, $actual, $message) { $pool('datasets', array_merge($read('pool/datasets'), array("tank/enc\tfilesystem", "backup/tank/enc\tfilesystem"))); $pool('marks', array_merge($read('pool/marks'), array("tank/enc@a\t4242\t9000"))); $pool('props', array_merge($read('pool/props'), array("tank/enc\tencryption\taes-256-gcm", "backup/tank/enc\tzfs.master:holder\ton"))); -same(array('can_proceed' => false, 'error' => 'backup/tank/enc only holds the copies of datasets below this one, and zfs cannot receive the raw copy of this encrypted dataset over it: copy it elsewhere instead, or move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all; a copy destroyed loses the snapshots only it had (one a repeat set in Replicate made asks for one Replicate by hand)'), +same(array('can_proceed' => false, 'error' => 'backup/tank/enc only holds the copies of datasets below this one, and zfs cannot receive the raw copy of this encrypted dataset over it: copy it elsewhere instead, or, as a copy destroyed loses the snapshots only it had, move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all (one a repeat set in Replicate made asks for one Replicate by hand)'), zfsmReplicatePlan('tank/enc', 'a', 'backup/tank/enc', array('estimate' => false)), 'a raw stream onto a holder'); -// Nor any copy onto an encrypted holder, as one under an encrypted dataset is (live, 2.4.4); one not read is not sent onto. +// Nor any copy onto an encrypted holder, as one under an encrypted dataset is; one not read is not sent onto. $pool('datasets', array_merge($read('pool/datasets'), array("tank/sealed\tfilesystem", "backup/tank/sealed\tfilesystem"))); $pool('marks', array_merge($read('pool/marks'), array("tank/sealed@a\t4646\t9004"))); $pool('props', array_merge($read('pool/props'), array("backup/tank/sealed\tzfs.master:holder\ton", "backup/tank/sealed\tencryption\taes-256-gcm"))); $plan = function() { return zfsmReplicatePlan('tank/sealed', 'a', 'backup/tank/sealed', array('estimate' => false)); }; -same(array('can_proceed' => false, 'error' => 'backup/tank/sealed only holds the copies of datasets below this one, and zfs cannot receive a copy over it, as it is encrypted: copy it elsewhere instead, or move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all; a copy destroyed loses the snapshots only it had (one a repeat set in Replicate made asks for one Replicate by hand)'), +same(array('can_proceed' => false, 'error' => 'backup/tank/sealed only holds the copies of datasets below this one, and zfs cannot receive a copy over it, as it is encrypted: copy it elsewhere instead, or, as a copy destroyed loses the snapshots only it had, move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all (one a repeat set in Replicate made asks for one Replicate by hand)'), $plan(), 'a stream onto an encrypted holder'); putenv('ZFSM_TEST_CRYPT_FAIL=backup/tank/sealed'); same(array('can_proceed' => false, 'error' => 'backup/tank/sealed only holds the copies of datasets below this one, and whether it is encrypted could not be read'), $plan(), 'one whose encryption could not be read'); putenv('ZFSM_TEST_CRYPT_FAIL'); -// Nor while a copy below it is mounted, which receiving over it would unmount (live, 2.4.4); one not read is not sent over. +// Nor while a copy below it is mounted, which receiving over it would unmount; one not read is not sent over. $pool('datasets', array_merge($read('pool/datasets'), array("tank/mnt\tfilesystem", "backup/tank/mnt\tfilesystem", "backup/tank/mnt/kid\tfilesystem"))); $pool('marks', array_merge($read('pool/marks'), array("tank/mnt@a\t4747\t9005"))); $pool('props', array_merge($read('pool/props'), array("backup/tank/mnt\tzfs.master:holder\ton", "backup/tank/mnt/kid\tmounted\tyes"))); $plan = function() { return zfsmReplicatePlan('tank/mnt', 'a', 'backup/tank/mnt', array('estimate' => false)); }; $over = 'backup/tank/mnt only holds the copies of datasets below this one, and '; -same(array('can_proceed' => false, 'error' => $over.'backup/tank/mnt/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted, the next run sends it'), $plan(), 'one with a copy below it mounted'); +same(array('can_proceed' => false, 'error' => $over.'backup/tank/mnt/kid below it is mounted, which receiving this copy over it would unmount: unmount it first'), $plan(), 'one with a copy below it mounted'); putenv('ZFSM_TEST_MOUNTED_FAIL=1'); same(array('can_proceed' => false, 'error' => $over.'whether a copy below it is mounted could not be read: cannot open \'backup/tank/mnt\': Input/output error'), $plan(), 'one where that could not be read'); putenv('ZFSM_TEST_MOUNTED_FAIL'); @@ -227,8 +233,16 @@ function same($expected, $actual, $message) { }; same(array('backup/tank/rf only holds the copies of datasets below this one, though this dataset made a copy there before; a scheduled run sends only changes, so replicate it once by hand to make a new copy', true, true), array($refill('tank/rf', true), $refill('tank/rf', false), $refill('tank/rf2', true)), 'a scheduled one where it made a copy is refused; by hand, or where it made none, it goes'); +// No holder is made for a dataset renamed or destroyed since a run listed it, nor where whether it is still there could not be read. +$placed = array(); +$gone = zfsmScheduleCopiesPlace('backup/', 'backup/tank/gp/sub', array(), $placed); +putenv('ZFSM_TEST_LIST_FAIL=tank/gp/sub'); +$unsure = zfsmScheduleCopiesPlace('backup/', 'backup/tank/gp/sub', array(), $placed); +putenv('ZFSM_TEST_LIST_FAIL'); +same(array('tank/gp/sub is gone, so no dataset was made to hold its copy', 'whether tank/gp/sub is still there could not be read, so no dataset was made to hold its copy', array(), array()), + array($gone, $unsure, $placed, array_values(preg_grep('~^backup/tank/gp[\t/]~', $read('pool/datasets')))), 'no holder for a dataset gone, or not read'); -// A first copy onto a holder that stops part-way carries on only with -F again (OpenZFS 2.4.4); the mark goes once it lands. +// A first copy onto a holder that stops part-way carries on only with -F again; the mark goes once it lands. $pool('datasets', array_merge($read('pool/datasets'), array("tank/res\tfilesystem", "backup/tank/res\tfilesystem"))); $pool('marks', array_merge($read('pool/marks'), array("tank/res@a\t4343\t9001"))); $pool('props', array_merge($read('pool/props'), array("backup/tank/res\tzfs.master:holder\ton"))); @@ -239,7 +253,7 @@ function same($expected, $actual, $message) { same(array(true, 'resume'), array(zfsmReplicaHolder(array('server' => ''), 'backup/tank/res'), zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false))['mode']), 'still a holder, part-received'); $pool('datasets', array_merge($read('pool/datasets'), array("backup/tank/res/kid\tfilesystem"))); $pool('props', array_merge($read('pool/props'), array("backup/tank/res/kid\tmounted\tyes"))); -same(array('can_proceed' => false, 'error' => 'backup/tank/res only holds the copies of datasets below this one, and backup/tank/res/kid below it is mounted, which receiving this copy over it would unmount; once it is unmounted, the next run sends it'), +same(array('can_proceed' => false, 'error' => 'backup/tank/res only holds the copies of datasets below this one, and backup/tank/res/kid below it is mounted, which receiving this copy over it would unmount: unmount it first'), zfsmReplicatePlan('tank/res', 'a', 'backup/tank/res', array('estimate' => false)), 'nor carried on over a copy below it mounted'); $pool('props', array_values(preg_grep("~^backup/tank/res/kid\tmounted\t~", $read('pool/props'), PREG_GREP_INVERT))); putenv('ZFSM_TEST_MARKS_FAIL=backup/tank/res'); diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index 6083a749..5fe4dd10 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -263,7 +263,7 @@ function same($expected, $actual, $message) { $under = preg_quote('tank/new/u#'.zfsmReplicaBookmarkPrefix('usb/tank/new/u').'auto-20270115080100 tank/new/u@', '~'); same(array(0, 1, 0), array(zfsmScheduleRun($usb, $cfg, $t + 3660), $sends('~^send -P -v -L -i '.$under.'~'), $sends('~^send -P -v -L tank/new/u@~')), 'and the one under usb, from its own'); same(true, zfsmScheduleRemove($usb, $cfg)['saved'], 'its copies under usb removed, which later checks leave out'); -// A run cut off part-way, then a rename: zfs finds the snapshot by guid under its new name (live, 2.4.4), so the next run carries it on. +// A run cut off part-way, then a rename: zfs finds the snapshot by guid under its new name, so the next run carries it on. putenv('ZFSM_TEST_SEND_CUT=1'); zfsmScheduleRun($id, $cfg, $t + 7200); putenv('ZFSM_TEST_SEND_CUT'); @@ -305,6 +305,16 @@ function same($expected, $actual, $message) { $add(array('backup/tank/again')); same(array($copy('tank/docker') => 'left where it is, as a dataset already has its new name'), renameDataset('tank/docker', 'tank/again', 0, $cfg)['failed'], 'a name already taken there'); same(array('succeeded' => array('tank/again' => 0), 'failed' => array()), renameDataset('tank/again', 'tank/again', 0, $cfg), 'a rename to its own name, which zfs does nothing for'); +// A rename that lands but can't be mounted there, which zfs 2.0 and 2.1 fail: its schedules follow it all the same, and the error stays. +$add(array('tank/um')); +same(true, $save('tank/um', $part('days', '02:10', '1'), $part('default'))['saved'], 'its own snapshots'); +putenv('ZFSM_TEST_RENAME_UNMOUNTED=tank/um'); +$unmounted = renameDataset('tank/um', 'tank/um2', 0, $cfg)['failed']; +putenv('ZFSM_TEST_RENAME_UNMOUNTED'); +$um = array_filter($stored(), function($job) { return zfsmUnderAny($job['dataset'], array('tank/um', 'tank/um2')); }); +same(array(array('tank/um2' => 'cannot mount \'/mnt/tank/um2\': mountpoint is not a directory'), array('tank/um2')), array($unmounted, array_values(array_column($um, 'dataset'))), + 'its schedule follows, and the mount error stays'); +foreach ($um as $job) zfsmScheduleRemove($job['id'], $cfg); // Its copy in a pool not imported now: zfs says the copy does not exist, so it is told as not checked; where no schedule copies it, // nothing is said. $add(array('tank/o')); @@ -350,6 +360,18 @@ function same($expected, $actual, $message) { same(array(true, true), array($save('tank/sealed/u2', $part('default'), $part('days', '05:20', '1', 0, 'usb'))['saved'], $save('tank/sealed/d5', $part('default'), $part('days', '06:20', '1', 0, 'backup'))['saved']), 'copies under usb above, and under backup below'); same(array(array(), true), array(renameDataset('tank/sealed/d5', 'tank/sealed/u2/d5', 0, $cfg)['failed'], in_array('backup/tank/sealed/u2/d5', $datasets(), true)), 'under one copied elsewhere, it moves'); foreach (array_filter($stored(), function($job) { return in_array($job['dataset'], array('tank/sealed/u2', 'tank/sealed/u2/d5'), true); }) as $job) zfsmScheduleRemove($job['id'], $cfg); +// A copy another schedule sends copies into stays where it is, so that one still finds them; a repeat set in Replicate of the renamed +// dataset itself, which keeps its old name, doesn't hold it. +$add(array('tank/nest', 'tank/nest/sub', 'backup/tank/nest', 'tank/nest3', 'backup/tank/nest3')); +$copied('tank/nest', 76); +$copied('tank/nest3', 77); +same(array(true, true), array($save('tank/nest/sub', $part('default'), $part('days', '06:40', '1', 0, 'backup/tank/nest'))['saved'], + zfsmScheduleChange(function($jobs) { return $jobs + array('3e5a7c9b' => array('id' => '3e5a7c9b', 'dataset' => 'tank/nest3', 'dest' => 'backup/tank/nest3', 'server' => '', 'every' => 'days') + zfsmScheduleAt('days', '03:30', '1') + array('keep' => 0)); })['saved']), + 'copies of the one below under its copy, and a repeat set in Replicate of another to its copy'); +same(array(array($copy('tank/nest') => 'left where it is, as a schedule sends copies into it (the copies of tank/nest2/sub and below), so the next copy of the renamed dataset is a full one'), true, array(), true), + array(renameDataset('tank/nest', 'tank/nest2', 0, $cfg)['failed'], in_array('backup/tank/nest', $datasets(), true), renameDataset('tank/nest3', 'tank/nest4', 0, $cfg)['failed'], in_array('backup/tank/nest4', $datasets(), true)), + 'a copy another Copy Under is in stays; one only its own stale repeat goes to moves'); +foreach (array_filter($stored(), function($job) { return in_array($job['dataset'], array('tank/nest2/sub', 'tank/nest3'), true); }) as $job) zfsmScheduleRemove($job['id'], $cfg); $add(array('tank/r', 'backup/tank/r', 'tank/q', 'backup/tank/q')); $copied('tank/q', 81); putenv('ZFSM_TEST_LIST_FAIL=backup/tank/r'); diff --git a/zfs.master/ZFSMasterSettings.page b/zfs.master/ZFSMasterSettings.page index cd5dc54b..220b974f 100644 --- a/zfs.master/ZFSMasterSettings.page +++ b/zfs.master/ZFSMasterSettings.page @@ -350,7 +350,7 @@ _(Replicate Every Dataset)_: > - A dataset's copy goes after the copy of the dataset above it. Where the dataset above one is not copied, an empty dataset, read-only and not mounted, is made in its place to hold the copies below it. If that dataset is copied later, its first copy replaces the empty one and the copies below it stay, though not while one of them is mounted, which receiving over it would unmount; an encrypted dataset sent raw can't replace it, nor can any copy replace an empty one that is encrypted, as one under an encrypted dataset is, and the run says what to do. When the copy of the dataset above one is missing because it failed, or the schedule that copies it has not made it yet, that one waits for the next run. > - **Snapshots To Keep**: for each dataset, as for a copy's schedule: only those it keeps of the snapshots taken for copies under its **Copy Under** stay, on the dataset and in its copy, whichever schedule took them: a dataset's own copies under the same one share them. Empty keeps all. A new **Copy Under** starts new copies there: the copies under the old one, and the snapshots taken for them, stay, as with Off. > - A dataset can have copies of its own, or None, under **Actions**, **Schedules** in the Main tab (**Replication**): it and the datasets below it follow that one instead, unless they have their own, each with its own **Copy Under**. -> - Renaming a dataset in the Main tab renames its copies under each **Copy Under** a schedule uses with it, unless a dataset already has the new name there, no schedule copies it or a dataset below it there now, what is there is neither its copy nor a holder of the copies below it, or a dataset made to hold it would stop an encrypted copy due there; it says which of those stayed, and a clash its schedules meet where they went. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves no room for the bookmark a copy made now keeps (named by Snapshots Pattern as it is now); a bookmark made before that no longer fits stays, and the rename says so. Copies under a **Copy Under** no schedule uses now stay as they are. Destroying a dataset removes what it and the datasets below it had under **Actions**, **Schedules**, and leaves its copies. +> - Renaming a dataset in the Main tab renames its copies under each **Copy Under** a schedule uses with it, unless a dataset already has the new name there, no schedule copies it or a dataset below it there now, another schedule sends copies into it, what is there is neither its copy nor a holder of the copies below it, or a dataset made to hold it would stop an encrypted copy due there; it says which of those stayed, and a clash its schedules meet where they went. Nothing is renamed while a copy of it or a dataset below it runs, or when the new name leaves no room for the bookmark a copy made now keeps (named by Snapshots Pattern as it is now); a bookmark made before that no longer fits stays, and the rename says so. Copies under a **Copy Under** no schedule uses now stay as they are. Destroying a dataset removes what it and the datasets below it had under **Actions**, **Schedules**, and leaves its copies. > - It and a snapshot schedule or copy's schedule of the same dataset can't start in the same minute: saving either one says so. > - A run waits for another replication of a dataset that is still sending; its waits add up to 55 minutes at most. A dataset that can't start in that time is skipped, and counts as a problem. A run that falls due while the one before still goes, as a first run can, is skipped and logged; once the one before has gone on for a day, it counts as a problem. > - Each run sends one notification at most: what it copied and its first problems. The first run with a problem sends it; later ones are logged in `/var/log/zfs.master.log` until a run works again, and that run's notification says so. The log has each dataset's replication. diff --git a/zfs.master/frontend/ZFSMImpact.js b/zfs.master/frontend/ZFSMImpact.js index ecc10c95..8f260e66 100644 --- a/zfs.master/frontend/ZFSMImpact.js +++ b/zfs.master/frontend/ZFSMImpact.js @@ -159,7 +159,7 @@ function zfsmScheduleDialogHtml(preview) { +say('None: no scheduled snapshots, or copies under Copy Under; a repeat set in Replicate still runs. Every hour or On days of the week: its own schedule.') +say('Datasets below this one follow its choice, unless they have their own.') +say('Replication copies this dataset and those below into Copy Under, a dataset on this server in another pool, each by its full name: tank/data goes to backup/tank/data. Each copy is read-only and unmounted; a run sends only the changes since its newest snapshot, or the whole dataset when it has no copy there yet.') - +say('Read-only datasets, such as copies received here, those mounted legacy, such as Docker\'s image layers, and those with Sanoid or Syncoid snapshots (autosnap_, syncoid_; Settings says when Snapshots Prefix counts instead) are never taken or copied. Schedules saves no repeat of either part on such a dataset itself, but a dataset below it that a run can take may have its own.') + +say('Read-only datasets, such as copies received here, those mounted legacy, such as Docker\'s image layers, and those with Sanoid or Syncoid snapshots (autosnap_, syncoid_; Settings says when Snapshots Prefix counts instead) are never taken or copied. Schedules saves no new or changed repeat of either part on such a dataset itself (one saved before stays, and its runs leave the dataset out), but a dataset below it that a run can take may have its own.') +say('Snapshots to keep: for Snapshots, after each run only the scheduled snapshots it keeps stay on each dataset, whichever snapshot schedule took them; for Replication, only those it keeps of the snapshots taken for copies under the same Copy Under stay, on each dataset and in its copy, whichever schedule took them, such as Replicate Every Dataset. A run removes up to 50 from each, and one held or cloned stays. '+zfsmKeepHelp()+' Snapshots made by hand are never removed.') +say('Settings lists every schedule.')+''; } @@ -178,7 +178,7 @@ function zfsmScheduleDialogSync(root) { function zfsmScheduleDialogChoices(root) { const copies = zfsmRepeatChoices(root, 'zfsm-copies'); return Object.assign(zfsmRepeatChoices(root, 'zfsm-schedules'), {copy_every: copies.every, copy_time: copies.time, copy_days: copies.days, copy_keep: copies.keep, - under: root.querySelector('#zfsm-copies-under').value.trim()}); + under: root.querySelector('#zfsm-copies-under').value}); } // What Copy goes to holds for a server: the last copy's dataset here, or its name inside the area there; another server's copy diff --git a/zfs.master/include/ZFSMReplicaPlan.php b/zfs.master/include/ZFSMReplicaPlan.php index 966080ad..cc72eb9d 100644 --- a/zfs.master/include/ZFSMReplicaPlan.php +++ b/zfs.master/include/ZFSMReplicaPlan.php @@ -134,7 +134,7 @@ function zfsmReplicatePlan($zdataset, $snapshot, $zdest, $with = array()) { } // $plan carrying on the run that stopped part-way and left $token on the copy (zfs send -t), whatever snapshot is picked now; onto -// a holder with -F again, which zfs asks for there (live, 2.4.4). Else why not. +// a holder with -F again, which zfs asks for there. Else why not. function zfsmReplicaPlanResume($plan, $target, $zdataset, $token) { $zdest = $plan['dest']; $where = zfsmReplicaTargetWhere($target, $zdest); @@ -157,7 +157,7 @@ function zfsmReplicaPlanResume($plan, $target, $zdataset, $token) { } // The snapshot zfs send -P -v -n -t $token sends, by the name zfs finds it under by guid; the token's toname keeps the one it had -// before a rename (live, 2.4.4). Null when $output names none. +// before a rename. Null when $output names none. function zfsmReplicaResumeSnapshot($output) { return preg_match('/^(?:full|incremental\t[^\t\n]+)\t([^\t\n]+)\t[0-9]+\s*$/m', (string)$output, $match) ? $match[1] : null; } @@ -184,13 +184,13 @@ function zfsmReplicaUnread($where, $why) { return 'The snapshots of '.$where.' could not be read'.($why !== '' ? ': '.$why : ''); } -// $plan into a copy that holds no snapshot: in full with -F when it is a holder, which keeps the copies below it (live, 2.4.4); zfs +// $plan into a copy that holds no snapshot: in full with -F when it is a holder, which keeps the copies below it; zfs // refuses a raw stream there, and any stream onto an encrypted holder. Else why not. function zfsmReplicaPlanHolder($plan, $target) { $where = zfsmReplicaTargetWhere($target, $plan['dest']); if (!zfsmReplicaHolder($target, $plan['dest'])) return $where.' already exists and holds no snapshot, so it can\'t be updated without destroying it'; $held = $where.' only holds the copies of datasets below this one, and '; - $fix = ': copy it elsewhere instead, or move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all; a copy destroyed loses the snapshots only it had (one a repeat set in Replicate made asks for one Replicate by hand)'; + $fix = ': copy it elsewhere instead, or, as a copy destroyed loses the snapshots only it had, move aside (zfs rename) each copy below it you want to keep, such as one of a dataset since destroyed, then destroy that dataset and the rest below it for the next run to send them all (one a repeat set in Replicate made asks for one Replicate by hand)'; if ($plan['raw']) return $held.'zfs cannot receive the raw copy of this encrypted dataset over it'.$fix; $crypt = getDatasetProperty('', $plan['dest'], 'encryption'); if ($crypt === null) return $held.'whether it is encrypted could not be read'; diff --git a/zfs.master/include/ZFSMReplicaTarget.php b/zfs.master/include/ZFSMReplicaTarget.php index 1ce1b64c..e2805880 100644 --- a/zfs.master/include/ZFSMReplicaTarget.php +++ b/zfs.master/include/ZFSMReplicaTarget.php @@ -42,13 +42,13 @@ function zfsmReplicaHolderClear($zdest) { return ($cleared['code'] ?? 1) === 0 ? '' : 'The copy still carries the mark of the empty dataset it replaced: '.zfsmCommandError($cleared, 'zfs inherit failed'); } -// Why a copy received over the holder $zdest would unmount one below it, which zfs leaves so (-u; live, 2.4.4); '' when none is. +// Why a copy received over the holder $zdest would unmount one below it, which zfs leaves so (-u); '' when none is. function zfsmReplicaHolderMounted($zdest) { $listed = runProcess('zfs', array('list', '-H', '-o', 'name,mounted', '-t', 'filesystem', '-r', $zdest)); if (($listed['code'] ?? 1) !== 0) return 'whether a copy below it is mounted could not be read: '.zfsmCommandError($listed, 'zfs list failed'); foreach (zfsmLines($listed['output']) as $line) { list($name, $mounted) = explode("\t", $line, 2) + array(1 => ''); - if ($mounted === 'yes') return $name.' below it is mounted, which receiving this copy over it would unmount; once it is unmounted, the next run sends it'; + if ($mounted === 'yes') return $name.' below it is mounted, which receiving this copy over it would unmount: unmount it first'; } return ''; } diff --git a/zfs.master/include/ZFSMSchedule.php b/zfs.master/include/ZFSMSchedule.php index c7dafbd8..07604669 100644 --- a/zfs.master/include/ZFSMSchedule.php +++ b/zfs.master/include/ZFSMSchedule.php @@ -146,6 +146,8 @@ function zfsmScheduleShare($a, $b, $jobs) { $near = zfsmScheduleFollowed($other['dataset'], array_diff_key($jobs, array($job['id'] => 0)) + array($job['id'] => $job), zfsmScheduleKindOf($job)); return $near['id'] === $job['id']; }; + // A run of copies leaves out the one dataset a repeat set in Replicate copies there (zfsmScheduleCopiesNot). + if (zfsmScheduleCopiesOwnRepeat($a, $b, $b['dataset']) || zfsmScheduleCopiesOwnRepeat($b, $a, $a['dataset'])) return false; return $a['dataset'] === $b['dataset'] || $in($a, $b) || $in($b, $a); } diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index c8c291cb..abae73d5 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -32,11 +32,16 @@ function zfsmScheduleCopiesNot($job, $jobs, $zdataset, $row, $name) { $dest = zfsmScheduleCopiesDest($job, $zdataset); if (!zfsmReplicaBookmarkFits($zdataset, zfsmReplicaTargetMark(array('server' => $job['server']), $dest), $name)) return 'the names of '.$zdataset.' and its copy are too long for the bookmark a copy keeps'; foreach ($jobs as $other) { - if ($other['dataset'] === $zdataset && $other['dest'] === $dest && $other['server'] === $job['server']) return $zdataset.' has a schedule of its own for that copy'; + if (zfsmScheduleCopiesOwnRepeat($job, $other, $zdataset)) return $zdataset.' has a schedule of its own for that copy'; } return ''; } +// Whether $other, a repeat set in Replicate, makes the copy of $zdataset that $job, copies under a dataset, would make. +function zfsmScheduleCopiesOwnRepeat($job, $other, $zdataset) { + return zfsmScheduleCopiesUnder($job) && $other['dataset'] === $zdataset && $other['dest'] === zfsmScheduleCopiesDest($job, $zdataset) && $other['server'] === $job['server']; +} + // The record Save makes of copies under $under here: Settings' Replicate Every Dataset ($zdataset ''), or a dataset's own; None // has none. function zfsmScheduleCopiesRecord($zdataset, $every, $under, $time, $days, $keep) { @@ -216,11 +221,15 @@ function zfsmScheduleCopiesLevels($dest, $zdest) { } // The datasets above $zdest, a copy under $dest: '' when each is there or made a holder, null to wait where a copy is $coming, else -// why a holder could not be made. $here gains each holder. +// why a holder could not be made, as for a dataset renamed or destroyed since a run listed it. $here gains each holder. function zfsmScheduleCopiesPlace($dest, $zdest, $coming, &$here) { + $zdataset = substr($zdest, strlen($dest)); + $missing = null; foreach (zfsmScheduleCopiesLevels($dest, $zdest) as $level) { if (isset($here[$level])) continue; if (isset($coming[$level])) return null; + $missing = $missing ?? zfsmDatasetMissing($zdataset); + if ($missing !== false) return ($missing ? $zdataset.' is gone' : 'whether '.$zdataset.' is still there could not be read').', so no dataset was made to hold its copy'; $why = zfsmReplicaHolderMake($level); if ($why !== '') return $why; $here[$level] = true; @@ -306,6 +315,8 @@ function zfsmScheduleCopiesMoveOne($job, $from, $to, $after, $cfg) { if ($pool !== false) return 'not checked, as '.($pool ? 'its pool is not imported now' : 'whether its pool is imported could not be read').': if it is there, it keeps its old name, and the next copy of the renamed dataset is a full one'; if ($missing === true) return ''; if ($missing === null) return 'not renamed with it, as whether it is there could not be read'; + $into = zfsmScheduleCopiesInto($from, $old, $after); + if ($into !== '') return 'left where it is, as a schedule sends copies into it ('.$into.'), so the next copy of the renamed dataset is a full one'; $whose = zfsmScheduleCopiesWhose($old, $to); if ($whose !== '') return $whose; $free = zfsmDatasetMissing($new); @@ -316,6 +327,16 @@ function zfsmScheduleCopiesMoveOne($job, $from, $to, $after, $cfg) { return $left === '' ? '' : 'renamed with it, but '.$left; } +// What of $jobs sends a copy here to $old or below it, which it would not find after a move; '' when none. A repeat set in Replicate +// of $from or below, which kept that name, finds no dataset to send anyway. +function zfsmScheduleCopiesInto($from, $old, $jobs) { + foreach ($jobs as $job) { + $dest = zfsmScheduleCopiesUnder($job) ? substr($job['dest'], 0, -1) : $job['dest']; + if ($job['server'] === '' && $dest !== '' && zfsmUnderAny($dest, array($old)) && !zfsmUnderAny($job['dataset'], array($from))) return zfsmScheduleWhat($job); + } + return ''; +} + // '' when the copy at $old is $to's: its newest snapshot is on $to (as zfsmReplicaPlanOnto finds it), or it is a holder; else why // it stays. function zfsmScheduleCopiesWhose($old, $to) { @@ -395,7 +416,7 @@ function zfsmScheduleCopiesKept($zdataset, $dest, $jobs) { } // zfs rename of copy $old to $new under $dest, after a holder is made where a level above $new is missing, as zfs rename needs its -// parent (live, 2.4.4), unless that holder would stop a copy $jobs make there (zfsmScheduleCopiesBlocked). '' or why not. +// parent, unless that holder would stop a copy $jobs make there (zfsmScheduleCopiesBlocked). '' or why not. function zfsmScheduleCopiesRename($dest, $old, $new, $jobs, $cfg) { $here = array(); foreach (zfsmScheduleCopiesLevels($dest, $new) as $level) { diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index 6371b560..31091df6 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -230,12 +230,11 @@ function zfsmScheduleOwnMove($from, $to) { return $changed + array('before' => $before, 'after' => $after); } -// $answer of zfs renaming $from to $to, or destroying it ($to ''), gaining what did not follow when that worked: its own schedules -// (zfsmScheduleOwnMove) and, on a rename only, a clash they meet there ($cfg names the snapshots) and its copies under each Copy -// Under (zfsmScheduleCopiesMove). +// Once zfs renamed $from to $to, or destroyed it ($to ''), as $answer says: its own schedules follow (zfsmScheduleOwnMove), and on a +// rename a clash they meet ($cfg names the snapshots) and its copies (zfsmScheduleCopiesMove). $answer gains each that did not. function zfsmScheduleOwnFollow($answer, $from, $to, $cfg = array()) { - // zfs renames a dataset to its own name by doing nothing. - if (!$answer['succeeded'] || $answer['failed'] || $from === $to) return $answer; + // zfs renames a dataset to its own name by doing nothing; 2.0 and 2.1 fail a rename that lands but can't mount or share it there. + if ($from === $to || ($answer['failed'] && !($to !== '' && zfsmDatasetGone($from) && zfsmDatasetMissing($to) === false))) return $answer; $moved = zfsmScheduleOwnMove($from, $to); $left = $to === '' ? 'not removed ('.$moved['why'].'): Remove them in Settings' : 'not moved to '.$to.' ('.$moved['why'].'): Remove them in Settings and set them again under Actions, Schedules'; From 78122731ef4a0ccfec18d1d6204765ad4437c56f Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:18:33 +0800 Subject: [PATCH 20/24] fix(schedules): a copy that moves but fails to mount is renamed with its dataset - zfs 2.0 and 2.1 exit 1 when a copy's rename lands but its remount fails: its bookmarks and holds now follow, and the error is shown - zfsmRenameLanded answers "did it move" for both the dataset's rename and its copies' - Test: a rename onto a name already taken leaves the schedules where they are --- tests/schedule_dialog_test.php | 14 +++++++++++++- zfs.master/backend/ZFSMOperations.php | 5 +++++ zfs.master/include/ZFSMScheduleCopies.php | 4 ++-- zfs.master/include/ZFSMScheduleSnapshot.php | 4 ++-- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index 5fe4dd10..d27cd676 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -306,8 +306,11 @@ function same($expected, $actual, $message) { same(array($copy('tank/docker') => 'left where it is, as a dataset already has its new name'), renameDataset('tank/docker', 'tank/again', 0, $cfg)['failed'], 'a name already taken there'); same(array('succeeded' => array('tank/again' => 0), 'failed' => array()), renameDataset('tank/again', 'tank/again', 0, $cfg), 'a rename to its own name, which zfs does nothing for'); // A rename that lands but can't be mounted there, which zfs 2.0 and 2.1 fail: its schedules follow it all the same, and the error stays. -$add(array('tank/um')); +$add(array('tank/um', 'tank/um-taken')); same(true, $save('tank/um', $part('days', '02:10', '1'), $part('default'))['saved'], 'its own snapshots'); +$under_um = function() use ($stored) { return array_values(array_column(array_filter($stored(), function($job) { return zfsmUnderAny($job['dataset'], array('tank/um', 'tank/um-taken')); }), 'dataset')); }; +same(array(array('tank/um-taken' => 'cannot rename \'tank/um\': dataset already exists'), array('tank/um')), array(renameDataset('tank/um', 'tank/um-taken', 0, $cfg)['failed'], $under_um()), + 'onto a name taken it fails where it is, and its schedule stays'); putenv('ZFSM_TEST_RENAME_UNMOUNTED=tank/um'); $unmounted = renameDataset('tank/um', 'tank/um2', 0, $cfg)['failed']; putenv('ZFSM_TEST_RENAME_UNMOUNTED'); @@ -372,6 +375,15 @@ function same($expected, $actual, $message) { array(renameDataset('tank/nest', 'tank/nest2', 0, $cfg)['failed'], in_array('backup/tank/nest', $datasets(), true), renameDataset('tank/nest3', 'tank/nest4', 0, $cfg)['failed'], in_array('backup/tank/nest4', $datasets(), true)), 'a copy another Copy Under is in stays; one only its own stale repeat goes to moves'); foreach (array_filter($stored(), function($job) { return in_array($job['dataset'], array('tank/nest2/sub', 'tank/nest3'), true); }) as $job) zfsmScheduleRemove($job['id'], $cfg); +// Its copy that moves but can't be mounted there, which zfs 2.0 and 2.1 fail, is renamed with it all the same: its bookmark too. +$add(array('tank/uc', 'backup/tank/uc')); +$copied('tank/uc', 78); +$pool('marks', array_merge($read('pool/marks'), array('tank/uc#'.zfsmReplicaBookmarkPrefix('backup/tank/uc')."s\t78\t500"))); +putenv('ZFSM_TEST_RENAME_UNMOUNTED=backup/tank/uc'); +$unmounted = renameDataset('tank/uc', 'tank/uc2', 0, $cfg)['failed']; +putenv('ZFSM_TEST_RENAME_UNMOUNTED'); +same(array(array($copy('tank/uc') => 'renamed with it, but cannot mount \'/mnt/backup/tank/uc2\': mountpoint is not a directory'), true, array('tank/uc2#'.zfsmReplicaBookmarkPrefix('backup/tank/uc2').'s')), + array($unmounted, in_array('backup/tank/uc2', $datasets(), true), array_values(preg_grep('~^tank/uc2#~', $marks()))), 'a copy that moves but is not mounted there is renamed, and says so'); $add(array('tank/r', 'backup/tank/r', 'tank/q', 'backup/tank/q')); $copied('tank/q', 81); putenv('ZFSM_TEST_LIST_FAIL=backup/tank/r'); diff --git a/zfs.master/backend/ZFSMOperations.php b/zfs.master/backend/ZFSMOperations.php index f8170599..a224911a 100644 --- a/zfs.master/backend/ZFSMOperations.php +++ b/zfs.master/backend/ZFSMOperations.php @@ -58,6 +58,11 @@ function zfsmDatasetGone($name) { return zfsmDatasetMissing($name) === true && zfsmDatasetMissing(strtok($name, '/')) === false; } +// After zfs rename failed: whether it moved $from to $to all the same, as 2.0 and 2.1 fail one they can't mount or share there. +function zfsmRenameLanded($from, $to) { + return zfsmDatasetGone($from) && zfsmDatasetMissing($to) === false; +} + // Whether a failed zfs command's output says its dataset does not exist. Only the closing diagnostic counts: a name may itself contain the words. function zfsmDatasetMissingSaid($output) { return preg_match('/: dataset does not exist\s*\z/', (string)$output) === 1; diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index abae73d5..30f4b3d0 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -322,8 +322,8 @@ function zfsmScheduleCopiesMoveOne($job, $from, $to, $after, $cfg) { $free = zfsmDatasetMissing($new); if ($free !== true) return $free === null ? 'not renamed with it, as whether its new name is free could not be read' : 'left where it is, as a dataset already has its new name'; $why = zfsmScheduleCopiesRename($job['dest'], $old, $new, $after, $cfg); - if ($why !== '') return 'not renamed with it: '.$why.'. It stays under its old name, so the next copy of the renamed dataset is a full one'; - $left = implode('; ', array_filter(array(zfsmScheduleCopiesMarks($old, $new, $to), zfsmScheduleCopiesHolds($old, $new, $to)), 'strlen')); + if ($why !== '' && !zfsmRenameLanded($old, $new)) return 'not renamed with it: '.$why.'. It stays under its old name, so the next copy of the renamed dataset is a full one'; + $left = implode('; ', array_filter(array($why, zfsmScheduleCopiesMarks($old, $new, $to), zfsmScheduleCopiesHolds($old, $new, $to)), 'strlen')); return $left === '' ? '' : 'renamed with it, but '.$left; } diff --git a/zfs.master/include/ZFSMScheduleSnapshot.php b/zfs.master/include/ZFSMScheduleSnapshot.php index 31091df6..56b5ddbd 100644 --- a/zfs.master/include/ZFSMScheduleSnapshot.php +++ b/zfs.master/include/ZFSMScheduleSnapshot.php @@ -233,8 +233,8 @@ function zfsmScheduleOwnMove($from, $to) { // Once zfs renamed $from to $to, or destroyed it ($to ''), as $answer says: its own schedules follow (zfsmScheduleOwnMove), and on a // rename a clash they meet ($cfg names the snapshots) and its copies (zfsmScheduleCopiesMove). $answer gains each that did not. function zfsmScheduleOwnFollow($answer, $from, $to, $cfg = array()) { - // zfs renames a dataset to its own name by doing nothing; 2.0 and 2.1 fail a rename that lands but can't mount or share it there. - if ($from === $to || ($answer['failed'] && !($to !== '' && zfsmDatasetGone($from) && zfsmDatasetMissing($to) === false))) return $answer; + // zfs renames a dataset to its own name by doing nothing. + if ($from === $to || ($answer['failed'] && !($to !== '' && zfsmRenameLanded($from, $to)))) return $answer; $moved = zfsmScheduleOwnMove($from, $to); $left = $to === '' ? 'not removed ('.$moved['why'].'): Remove them in Settings' : 'not moved to '.$to.' ('.$moved['why'].'): Remove them in Settings and set them again under Actions, Schedules'; From ef9800d60d9e0048a5970980055989b9af6b9be8 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:33:34 +0800 Subject: [PATCH 21/24] fix(schedules): no holder above a copy whose names were not read; a rename's fit check follows the new parent - A dataset whose snapshot names were not read may be due: the copy below waits instead of making a holder that would refuse it - A holder another run made since this one listed is used, not reported as a failed create - The rename's fit check reads a read-only or legacy mount from the new parent where it may be inherited, and sees every dataset below - A copy left where it is because its new name is taken says so, without predicting a full send --- tests/schedule_copies_test.php | 17 ++++++++ tests/schedule_dialog_test.php | 22 ++++++++++- zfs.master/backend/ZFSMOperations.php | 7 ++-- zfs.master/include/ZFSMScheduleCopies.php | 48 +++++++++++++++++------ 4 files changed, 79 insertions(+), 15 deletions(-) diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index 703af3ff..a3e4ad3c 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -241,6 +241,11 @@ function same($expected, $actual, $message) { putenv('ZFSM_TEST_LIST_FAIL'); same(array('tank/gp/sub is gone, so no dataset was made to hold its copy', 'whether tank/gp/sub is still there could not be read, so no dataset was made to hold its copy', array(), array()), array($gone, $unsure, $placed, array_values(preg_grep('~^backup/tank/gp[\t/]~', $read('pool/datasets')))), 'no holder for a dataset gone, or not read'); +// One another run made since this one listed is there to copy into; a create that fails where none is still says why. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/ar\tfilesystem", "tank/ar/slow\tfilesystem", "backup/tank/ar\tfilesystem"))); +$placed = array('backup/tank' => true, 'gone/tank' => true); +same(array('', 'The dataset to hold the copies below gone/tank/ar could not be made: cannot create \'gone/tank/ar\': parent does not exist'), + array(zfsmScheduleCopiesPlace('backup/', 'backup/tank/ar/slow', array(), $placed), zfsmScheduleCopiesPlace('gone/', 'gone/tank/ar/slow', array(), $placed)), 'a parent made since'); // A first copy onto a holder that stops part-way carries on only with -F again; the mark goes once it lands. $pool('datasets', array_merge($read('pool/datasets'), array("tank/res\tfilesystem", "backup/tank/res\tfilesystem"))); @@ -417,4 +422,16 @@ function same($expected, $actual, $message) { array(zfsmScheduleRun($id, $cfg, $t + 36000), zfsmScheduleRun('cccc0000', $cfg, $t + 36060), array_values(preg_grep('~^backup/~', $datasets())), array_values(preg_grep('/^create /', $read('calls.log')))), 'its copy, then a holder for tank/data, which nothing copies, and the copy below'); +// Nor above one where the snapshot names were read only in part: tank and tank/data, which have none, may be due, so the copy waits. +$pool('datasets', array("tank\tfilesystem", "tank/bad\tfilesystem", "tank/data\tfilesystem", "tank/data/sub\tfilesystem", "backup\tfilesystem")); +$pool('marks', array("tank/data/sub@a\t5001\t9001")); +file_put_contents(zfsmScheduleFile(), json_encode(array(array('keep' => 1) + $every))); +$reset(); +@unlink(zfsmScheduleStateFile($id)); +putenv('ZFSM_TEST_COVER_FAIL=tank/bad'); +$part_read = zfsmScheduleRun($id, $cfg, $t + 39600); +putenv('ZFSM_TEST_COVER_FAIL'); +same(array(1, array(), array()), array($part_read, preg_grep('~^backup/~', $datasets()), preg_grep('/^create /', $read('calls.log'))), 'no holder where the names were read in part'); +same(1, preg_match('/Not copied, as the copy of a dataset above it is missing \(tank\/data\/sub\)/', $read('notify-detail.log')[0] ?? ''), 'it waits'); + echo "schedule copies tests passed\n"; diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index d27cd676..e651a8bc 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -374,7 +374,12 @@ function same($expected, $actual, $message) { same(array(array($copy('tank/nest') => 'left where it is, as a schedule sends copies into it (the copies of tank/nest2/sub and below), so the next copy of the renamed dataset is a full one'), true, array(), true), array(renameDataset('tank/nest', 'tank/nest2', 0, $cfg)['failed'], in_array('backup/tank/nest', $datasets(), true), renameDataset('tank/nest3', 'tank/nest4', 0, $cfg)['failed'], in_array('backup/tank/nest4', $datasets(), true)), 'a copy another Copy Under is in stays; one only its own stale repeat goes to moves'); -foreach (array_filter($stored(), function($job) { return in_array($job['dataset'], array('tank/nest2/sub', 'tank/nest3'), true); }) as $job) zfsmScheduleRemove($job['id'], $cfg); +// Where a dataset has its new name already, that is why it stays: the next copy may go there as changes alone. +$add(array('tank/nest6', 'tank/nest6/sub', 'backup/tank/nest6', 'backup/tank/nest7')); +$copied('tank/nest6', 79); +same(true, $save('tank/nest6/sub', $part('default'), $part('days', '06:50', '1', 0, 'backup/tank/nest6'))['saved'], 'copies of the one below under its copy'); +same(array($copy('tank/nest6') => 'left where it is, as a dataset already has its new name'), renameDataset('tank/nest6', 'tank/nest7', 0, $cfg)['failed'], 'its new name taken'); +foreach (array_filter($stored(), function($job) { return in_array($job['dataset'], array('tank/nest2/sub', 'tank/nest3', 'tank/nest7/sub'), true); }) as $job) zfsmScheduleRemove($job['id'], $cfg); // Its copy that moves but can't be mounted there, which zfs 2.0 and 2.1 fail, is renamed with it all the same: its bookmark too. $add(array('tank/uc', 'backup/tank/uc')); $copied('tank/uc', 78); @@ -420,6 +425,21 @@ function same($expected, $actual, $message) { putenv('ZFSM_TEST_COVER_FAIL'); same(array(array($longer('tank/sz') => 0, $longer('tank/ro') => 0, $longer('tank/lg') => 0), array($longer('tank/un') => 'Nothing was renamed: the names of '.$longer('tank/un').' and its copy would be too long for the bookmark a copy keeps; pick a shorter name')), array($renamed, $unread), 'one a run leaves out may take a long name; one not read may not'); +// One whose read-only or legacy mount may come from above it is checked under its new parent; one that sets its own is not. +$add(array('tank/rp', 'tank/rp/c', 'tank/lp', 'tank/lp/c', 'tank/rq', 'tank/rq/c')); +$pool('props', array_merge($read('pool/props'), array("tank/rp\treadonly\ton", "tank/rp/c\treadonly\ton", "tank/lp\tmountpoint\tlegacy", "tank/lp/c\tmountpoint\tlegacy", "tank/rq/c\treadonly\ton"))); +$moved = array(); +foreach (array('tank/rp/c' => 'tank/rpc', 'tank/lp/c' => 'tank/lpc', 'tank/rq/c' => 'tank/rqc') as $zdataset => $to) { + $answer = renameDataset($zdataset, $longer($to), 0, $cfg); + $moved += $answer['failed'] + $answer['succeeded']; +} +$too = function($zdataset) { return 'Nothing was renamed: the names of '.$zdataset.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'; }; +same(array($longer('tank/rpc') => $too($longer('tank/rpc')), $longer('tank/lpc') => $too($longer('tank/lpc')), $longer('tank/rqc') => 0), $moved, + 'one moved out from under a read-only or legacy parent is checked; one read-only itself is not'); +putenv('ZFSM_TEST_COVER_FAIL=tank'); +$unsure = renameDataset($longer('tank/rqc'), $longer('tank/rqd'), 0, $cfg)['failed']; +putenv('ZFSM_TEST_COVER_FAIL'); +same(array($longer('tank/rqd') => $too($longer('tank/rqd'))), $unsure, 'one whose parents could not be read is checked'); // Its own copies go with it, so the room is that under their Copy Under. $far = 'backup/'.str_repeat('c', 20); $kin = str_repeat('n', 50); diff --git a/zfs.master/backend/ZFSMOperations.php b/zfs.master/backend/ZFSMOperations.php index a224911a..b1b4fe88 100644 --- a/zfs.master/backend/ZFSMOperations.php +++ b/zfs.master/backend/ZFSMOperations.php @@ -637,10 +637,11 @@ function zfsmSetShareOwner($pool, $names) { function renameDataset($zdataset, $new_name, $force, $cfg) { if (!zfsmDatasetExists($zdataset) || !zfsmValidDatasetName($new_name)) return zfsmFailure($new_name, 'Invalid source or destination dataset'); // Held until its copies follow it, so no run sends it under its new name first. - $locks = zfsmScheduleCopiesHold($zdataset); - if (is_string($locks)) return zfsmFailure($new_name, 'Nothing was renamed: '.$locks); + $held = zfsmScheduleCopiesHold($zdataset); + if (is_string($held)) return zfsmFailure($new_name, 'Nothing was renamed: '.$held); + list($locks, $names) = $held; try { - $why = zfsmScheduleCopiesFit(array_keys($locks), $zdataset, $new_name, $cfg); + $why = zfsmScheduleCopiesFit($names, $zdataset, $new_name, $cfg); if ($why !== '') return zfsmFailure($new_name, 'Nothing was renamed: '.$why); $args = array('rename'); if ((string)$force === '1') $args[] = '-f'; diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index 30f4b3d0..1a95d91c 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -176,6 +176,8 @@ function zfsmScheduleCopiesComing($job, $jobs, $listed) { $coming = array(); foreach (zfsmScheduleListed($listed)[0] as $zdataset => $row) { $by = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); + // One whose snapshot names were not read may be due too. + $row[2] = $row[2] ?? false; if (($by['dest'] ?? '') === $job['dest'] && zfsmScheduleLeftOut($by, $jobs, $zdataset, $row, $listed['name']) === '') $coming[zfsmScheduleCopiesDest($by, $zdataset)] = true; } foreach (zfsmScheduleCopiesRepeatsTo($jobs) as $zdest => $from) $coming[$zdest] = true; @@ -231,7 +233,8 @@ function zfsmScheduleCopiesPlace($dest, $zdest, $coming, &$here) { $missing = $missing ?? zfsmDatasetMissing($zdataset); if ($missing !== false) return ($missing ? $zdataset.' is gone' : 'whether '.$zdataset.' is still there could not be read').', so no dataset was made to hold its copy'; $why = zfsmReplicaHolderMake($level); - if ($why !== '') return $why; + // Another run may have made it since this one listed. + if ($why !== '' && !zfsmDatasetExists($level, 'filesystem')) return $why; $here[$level] = true; } return ''; @@ -244,11 +247,16 @@ function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { // The rename's follow-up says so. if ($read === null) return ''; $snapshot = zfsmNewSnapshotName($cfg, time()); - // One a run leaves out (zfsmScheduleSkipped) needs no room; one whose row could not be read is checked. Read once a name is too long. - $rows = null; - $skipped = function($zdataset, $name) use (&$rows, $cfg, $from) { - if ($rows === null) $rows = zfsmScheduleListed(zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-r', $from)))[0]; - return isset($rows[$name]) && zfsmScheduleSkipped($zdataset, $rows[$name]) !== ''; + // One a run leaves out (zfsmScheduleSkipped), there and once moved, needs no room; one whose row could not be read is checked. Read + // once a name is too long. + $rows = $parents = null; + $skipped = function($zdataset, $name) use (&$rows, &$parents, $cfg, $from, $to) { + if ($rows === null) { + $rows = zfsmScheduleListed(zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-r', $from)))[0]; + $parents = zfsmScheduleCopiesParents($from, $to); + } + if (!isset($rows[$name])) return false; + return zfsmScheduleSkipped($zdataset, $zdataset === $name ? $rows[$name] : zfsmScheduleCopiesMovedRow($rows[$name], ...$parents)) !== ''; }; $fits = function($zdataset, $jobs, $name) use ($snapshot, $skipped) { $job = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); @@ -264,17 +272,35 @@ function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { return ''; } +// The rows (zfsmScheduleListed) of the datasets above $from and $to, each null when not read. +function zfsmScheduleCopiesParents($from, $to) { + $up = array_map(function($zdataset) { return substr($zdataset, 0, (int)strrpos($zdataset, '/')); }, array($from, $to)); + $listed = runProcess('zfs', array_merge(ZFSM_SCHEDULE_COVERED, $up)); + $rows = ($listed['code'] ?? 1) === 0 ? zfsmScheduleListed(array('output' => $listed['output']))[0] : array(); + return array($rows[$up[0]] ?? null, $rows[$up[1]] ?? null); +} + +// $row of a dataset at or below one moved from under $old to under $new (zfsmScheduleCopiesParents): a read-only or legacy mount it +// may take from $old takes $new's. +function zfsmScheduleCopiesMovedRow($row, $old, $new) { + foreach (array('on', 'legacy') as $i => $skip) { + if ($row[$i] === $skip && ($old[$i] ?? $skip) === $skip) $row[$i] = $new[$i] ?? ''; + } + return $row; +} + // The replication locks (zfsmJobLock, no command started holds them) of $zdataset and each dataset below it a run can copy, by -// name; why not when one is held or can't be had. +// name, and the names of all of them; why not when one is held or can't be had. function zfsmScheduleCopiesHold($zdataset) { $listed = runProcess('zfs', array('list', '-H', '-o', 'name,guid,mountpoint', '-t', 'filesystem,volume', '-r', $zdataset)); if (($listed['code'] ?? 1) !== 0) return 'the datasets at and below it could not be read: '.zfsmCommandError($listed, 'zfs list failed'); $read = zfsmScheduleRead(); $repeats = array(); foreach ($read === null ? array() : $read['valid'] as $job) if (!zfsmScheduleCovers($job)) $repeats[$job['dataset']] = true; - $locks = array(); + $locks = $names = array(); foreach (zfsmLines($listed['output']) as $line) { list($name, $guid, $mountpoint) = explode("\t", $line, 3) + array('', '', ''); + $names[] = $name; $file = zfsmJobGuidLock($guid, 'replicate'); // Docker's legacy-mounted image layers, often thousands, which Copy Under skips: a lock each held could run out of open files, so // one no repeat copies is only checked, where a run made its lock. @@ -288,7 +314,7 @@ function zfsmScheduleCopiesHold($zdataset) { if ($check) zfsmJobUnlock($lock); else $locks[$name] = $lock; } - return $locks; + return array($locks, $names); } // After zfs renamed $from to $to and its own schedules followed ($before, $after): its copy under each dataset that copies go under @@ -315,12 +341,12 @@ function zfsmScheduleCopiesMoveOne($job, $from, $to, $after, $cfg) { if ($pool !== false) return 'not checked, as '.($pool ? 'its pool is not imported now' : 'whether its pool is imported could not be read').': if it is there, it keeps its old name, and the next copy of the renamed dataset is a full one'; if ($missing === true) return ''; if ($missing === null) return 'not renamed with it, as whether it is there could not be read'; - $into = zfsmScheduleCopiesInto($from, $old, $after); - if ($into !== '') return 'left where it is, as a schedule sends copies into it ('.$into.'), so the next copy of the renamed dataset is a full one'; $whose = zfsmScheduleCopiesWhose($old, $to); if ($whose !== '') return $whose; $free = zfsmDatasetMissing($new); if ($free !== true) return $free === null ? 'not renamed with it, as whether its new name is free could not be read' : 'left where it is, as a dataset already has its new name'; + $into = zfsmScheduleCopiesInto($from, $old, $after); + if ($into !== '') return 'left where it is, as a schedule sends copies into it ('.$into.'), so the next copy of the renamed dataset is a full one'; $why = zfsmScheduleCopiesRename($job['dest'], $old, $new, $after, $cfg); if ($why !== '' && !zfsmRenameLanded($old, $new)) return 'not renamed with it: '.$why.'. It stays under its old name, so the next copy of the renamed dataset is a full one'; $left = implode('; ', array_filter(array($why, zfsmScheduleCopiesMarks($old, $new, $to), zfsmScheduleCopiesHolds($old, $new, $to)), 'strlen')); From 70c94774ab505de92f36703775723f2c01d0e2b3 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:33:04 +0800 Subject: [PATCH 22/24] test(schedules): the rename-unmounted setup saves at a minute the hourly copies never use - tank/um's Monday snapshots were at 02:10, which clashes with Replicate Every Dataset when its random minute is 10 --- tests/schedule_dialog_test.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index e651a8bc..84ec82d2 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -307,7 +307,7 @@ function same($expected, $actual, $message) { same(array('succeeded' => array('tank/again' => 0), 'failed' => array()), renameDataset('tank/again', 'tank/again', 0, $cfg), 'a rename to its own name, which zfs does nothing for'); // A rename that lands but can't be mounted there, which zfs 2.0 and 2.1 fail: its schedules follow it all the same, and the error stays. $add(array('tank/um', 'tank/um-taken')); -same(true, $save('tank/um', $part('days', '02:10', '1'), $part('default'))['saved'], 'its own snapshots'); +same(true, $save('tank/um', $part('days', $free, '1'), $part('default'))['saved'], 'its own snapshots'); $under_um = function() use ($stored) { return array_values(array_column(array_filter($stored(), function($job) { return zfsmUnderAny($job['dataset'], array('tank/um', 'tank/um-taken')); }), 'dataset')); }; same(array(array('tank/um-taken' => 'cannot rename \'tank/um\': dataset already exists'), array('tank/um')), array(renameDataset('tank/um', 'tank/um-taken', 0, $cfg)['failed'], $under_um()), 'onto a name taken it fails where it is, and its schedule stays'); From 3bea43a8b32dd1e2f5d28c8fec002d0557e4ce75 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:10:12 +0800 Subject: [PATCH 23/24] fix(schedules): no holder where a schedule saved mid-run copies; the fit check follows inherited readonly and mountpoint - Before a run makes a holder it reads the schedules again: where one copies that dataset there now, or they can't be read, the copy below waits - A rename's bookmark-length check reads where readonly and mountpoint come from: a default or inherited value takes the new parent's, its own stays - Coming no longer counts a dataset with unread snapshot names as due: the fresh read waits on it - Test fake: zfs list inherits readonly and mountpoint in both its reads; zfs get -o source says where each comes from - Tests: a resumed run lets go of its holds; a received copy lands read-only; two comments cut to two lines --- tests/fixtures/bin/zfs | 26 ++++++-- tests/schedule_copies_test.php | 21 +++++++ tests/schedule_dialog_test.php | 26 ++++++-- zfs.master/include/ZFSMScheduleCopies.php | 75 ++++++++++++++--------- 4 files changed, 108 insertions(+), 40 deletions(-) diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index fe3a527c..c7c5f1db 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -29,9 +29,15 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n begin() { printf '\000\000\000\000\000\000\000\000\254\313\272\365\002\000\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\002\000\000\000\000\000\000\000'; head -c 272 /dev/zero; } newest() { awk -F'\t' -v d="$1@" 'index($1, d) == 1 { g = $2 } END { print g }' "$R/marks"; } held() { awk -F'\t' -v n="$1" -v t="$2" '$1 == n && (t == "" || $2 == t) { f = 1 } END { exit !f }' "$R/holds"; } + # The nearest of $1 and those above it that sets property $2, or nothing. + setter() { s=$1; while :; do awk -F'\t' -v n="$s" -v p="$2" '$1 == n && $2 == p { f = 1 } END { exit !f }' "$R/props" && { echo "$s"; return; }; [ "${s%/*}" = "$s" ] && return; s=${s%/*}; done; } + # zfs list's $2 of $1: its setter's (a mountpoint path gains the rest of the name), else $3. + inherited() { s=$(setter "$1" "$2"); [ -n "$s" ] || { echo "$3"; return; }; v=$(prop "$s" "$2" ''); case "$2:$v" in mountpoint:/*) echo "$v${1#"$s"}" ;; *) echo "$v" ;; esac; } + # zfs get -o source of $1's $2. + source_of() { s=$(setter "$1" "$2"); if [ -z "$s" ]; then echo default; elif [ "$s" = "$1" ]; then echo local; else echo "inherited from $s"; fi; } case "$1" in list) - # A snapshot schedule's dataset read (include/ZFSMScheduleSnapshot.php): readonly, mountpoint from props, else off, /mnt/ (a volume -); + # A snapshot schedule's dataset read (include/ZFSMScheduleSnapshot.php): readonly, mountpoint set or inherited, else off, /mnt/ (a volume -); # -d 1 : it and its children. ZFSM_TEST_COVER_FAIL: 1 fails it, hang waits 30 s, a dataset's name fails that one, then exit 1. if contains_arg 'name,readonly,mountpoint' "$@"; then [ "${ZFSM_TEST_COVER_FAIL:-0}" = 1 ] && { printf 'cannot iterate filesystems: Input/output error\n' >&2; exit 1; } @@ -43,8 +49,8 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n while IFS="$(printf '\t')" read -r n ty; do if [ -n "$under" ]; then case "$n" in "$under") ;; "$under"/*/*) [ "$deep" = 1 ] || continue ;; "$under"/*) ;; *) continue ;; esac; fi [ "$n" = "${ZFSM_TEST_COVER_FAIL:-}" ] && { printf "cannot open '%s': Input/output error\n" "$n" >&2; code=1; continue; } - if [ "$ty" = volume ]; then mp=-; else mp=/mnt/$n; fi - printf '%s\t%s\t%s\n' "$n" "$(prop "$n" readonly off)" "$(prop "$n" mountpoint "$mp")" + if [ "$ty" = volume ]; then mp=-; else mp=$(inherited "$n" mountpoint "/mnt/$n"); fi + printf '%s\t%s\t%s\n' "$n" "$(inherited "$n" readonly off)" "$mp" done < "$R/datasets" exit $code fi @@ -116,8 +122,8 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n [ "${ZFSM_TEST_TREE_FAIL:-0}" = 1 ] && { printf "cannot open '%s': Input/output error\n" "$last_arg" >&2; exit 1; } exists "$last_arg" || { printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1; } awk -F'\t' -v d="$last_arg" '$1 == d || index($1, d "/") == 1' "$R/datasets" | while IFS="$(printf '\t')" read -r n ty; do - if [ "$ty" = volume ]; then mp=-; else mp=/mnt/$n; fi - printf '%s\t%s\t%s\n' "$n" "$(prop "$n" guid "$(printf '%s' "$n" | cksum | cut -d' ' -f1)")" "$(prop "$n" mountpoint "$mp")" + if [ "$ty" = volume ]; then mp=-; else mp=$(inherited "$n" mountpoint "/mnt/$n"); fi + printf '%s\t%s\t%s\n' "$n" "$(prop "$n" guid "$(printf '%s' "$n" | cksum | cut -d' ' -f1)")" "$mp" done exit 0 fi @@ -137,6 +143,16 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n exit 0 ;; esac exists "$last_arg" || { printf "cannot open '%s': dataset does not exist\n" "$last_arg" >&2; exit 1; } + # A rename's read of where readonly and mountpoint come from, at the dataset and below (a volume's mountpoint is -); + # ZFSM_TEST_SOURCE_FAIL=1 fails it. + if [ "$4" = name,property,source ]; then + [ "${ZFSM_TEST_SOURCE_FAIL:-0}" = 1 ] && { printf "cannot get property for '%s': Input/output error\n" "$last_arg" >&2; exit 1; } + awk -F'\t' -v d="$last_arg" '$1 == d || index($1, d "/") == 1' "$R/datasets" | while IFS="$(printf '\t')" read -r n ty; do + printf '%s\treadonly\t%s\n' "$n" "$(source_of "$n" readonly)" + if [ "$ty" = volume ]; then printf '%s\tmountpoint\t-\n' "$n"; else printf '%s\tmountpoint\t%s\n' "$n" "$(source_of "$n" mountpoint)"; fi + done + exit 0 + fi # -o source: local where it was set, else inherited from the nearest dataset above that set it, as a user property is. if [ "$5" = source ]; then d=$last_arg diff --git a/tests/schedule_copies_test.php b/tests/schedule_copies_test.php index a3e4ad3c..45eebc72 100644 --- a/tests/schedule_copies_test.php +++ b/tests/schedule_copies_test.php @@ -246,6 +246,16 @@ function same($expected, $actual, $message) { $placed = array('backup/tank' => true, 'gone/tank' => true); same(array('', 'The dataset to hold the copies below gone/tank/ar could not be made: cannot create \'gone/tank/ar\': parent does not exist'), array(zfsmScheduleCopiesPlace('backup/', 'backup/tank/ar/slow', array(), $placed), zfsmScheduleCopiesPlace('gone/', 'gone/tank/ar/slow', array(), $placed)), 'a parent made since'); +// Nor where a schedule copies the parent now, as one saved since the run read them: the copy below waits; with none, the holder is made. +$pool('datasets', array_merge($read('pool/datasets'), array("tank/cv\tfilesystem", "tank/cv/kid\tfilesystem"))); +$jobs_then = file_get_contents(zfsmScheduleFile()); +file_put_contents(zfsmScheduleFile(), json_encode(array($every))); +$due = zfsmScheduleCopiesPlace('backup/', 'backup/tank/cv/kid', array(), $placed, $cfg); +$made_then = in_array('backup/tank/cv', $datasets(), true); +file_put_contents(zfsmScheduleFile(), json_encode(array())); +same(array(null, false, '', true), array($due, $made_then, zfsmScheduleCopiesPlace('backup/', 'backup/tank/cv/kid', array(), $placed, $cfg), in_array('backup/tank/cv', $datasets(), true)), + 'one a schedule copies now waits; one none copies gets its holder'); +file_put_contents(zfsmScheduleFile(), $jobs_then); // A first copy onto a holder that stops part-way carries on only with -F again; the mark goes once it lands. $pool('datasets', array_merge($read('pool/datasets'), array("tank/res\tfilesystem", "backup/tank/res\tfilesystem"))); @@ -434,4 +444,15 @@ function same($expected, $actual, $message) { same(array(1, array(), array()), array($part_read, preg_grep('~^backup/~', $datasets()), preg_grep('/^create /', $read('calls.log'))), 'no holder where the names were read in part'); same(1, preg_match('/Not copied, as the copy of a dataset above it is missing \(tank\/data\/sub\)/', $read('notify-detail.log')[0] ?? ''), 'it waits'); +// Nor where Replicate Every Dataset copies it since the run read the schedules: tank/data's Replication None removed, the copy waits. +$pool('datasets', array("tank\tfilesystem", "tank/data\tfilesystem", "tank/data/sub\tfilesystem", "backup\tfilesystem", "backup/tank\tfilesystem")); +$then = zfsmScheduleRead(json_encode(array(array('keep' => 1) + $every, array('id' => 'dddd0000', 'dataset' => 'tank/data', 'dest' => '/', 'every' => 'none') + $every, $under)))['valid']; +file_put_contents(zfsmScheduleFile(), json_encode(array(array('keep' => 1) + $every, $under))); +$reset(); +zfsmScheduleCopiesMade($then['cccc0000'], $then, $cfg, $t + 43200, ZFSM_SCHEDULE_WAIT); +same(array(array(), array()), array(preg_grep('~^backup/tank/~', $datasets()), preg_grep('/^create /', $read('calls.log'))), 'no holder where a schedule saved since copies'); +file_put_contents(zfsmScheduleFile(), '{'); +$placed = array(); +same(null, zfsmScheduleCopiesPlace('backup/', 'backup/tank/data/sub', array(), $placed, $cfg), 'nor where the schedules saved could not be read'); + echo "schedule copies tests passed\n"; diff --git a/tests/schedule_dialog_test.php b/tests/schedule_dialog_test.php index 84ec82d2..767f2a7f 100644 --- a/tests/schedule_dialog_test.php +++ b/tests/schedule_dialog_test.php @@ -255,6 +255,9 @@ function same($expected, $actual, $message) { // Its snapshots gone from the source, only the bookmarks that took the new names keep the next run incremental. $pool('marks', array_values(preg_grep('~^tank/new(/sub)?@auto-20270115080000\t~', $read('pool/marks'), PREG_GREP_INVERT))); same(0, zfsmScheduleRun($id, $cfg, $t + 3600), 'the next run'); +$landed = zfsmScheduleListed(zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-d', '1', 'backup/tank')))[0]['backup/tank'] ?? null; +same('backup/tank is read-only, as a copy received here is: a snapshot in it stops the next replication into it', zfsmScheduleSkipped('backup/tank', $landed), + 'a copy lands read-only, so no snapshot schedule takes it'); $from = function($zdataset) { return preg_quote($zdataset.'#'.zfsmReplicaBookmarkPrefix('backup/'.$zdataset).'auto-20270115080000 '.$zdataset.'@', '~'); }; same(array(1, 1, 0), array($sends('~^send -P -v -L -i '.$from('tank/new').'~'), $sends('~^send -P -v -L -i '.$from('tank/new/sub').'~'), $sends('~^send -P -v -L tank/new@~')), 'sends from the moved bookmarks only the changes to the renamed copies'); @@ -267,6 +270,8 @@ function same($expected, $actual, $message) { putenv('ZFSM_TEST_SEND_CUT=1'); zfsmScheduleRun($id, $cfg, $t + 7200); putenv('ZFSM_TEST_SEND_CUT'); +$pinned = function($zdataset) use ($read) { return array_values(preg_grep('~^'.preg_quote($zdataset, '~').'(/sub)?@auto-20270115100000\t'.ZFSM_KEEP_HOLD.'~', $read('pool/holds'))); }; +same(2, count($pinned('tank/new')), 'the cut-off run holds the snapshots it sends'); $tokens = function($copy) use ($read) { return array_values(preg_grep('~^'.preg_quote($copy, '~').'(/sub)?\treceive_resume_token\ttok:~', $read('pool/props'))); }; same(array(2, array('succeeded' => array('tank/cut' => 0), 'failed' => array())), array(count($tokens('backup/tank/new')), renameDataset('tank/new', 'tank/cut', 0, $cfg)), 'a rename after a run cut off part-way'); @@ -274,6 +279,7 @@ function same($expected, $actual, $message) { $marks = function() use ($read) { return array_map(function($line) { return explode("\t", $line)[0]; }, $read('pool/marks')); }; same(array(0, 2, array(), array()), array(zfsmScheduleRun($id, $cfg, $t + 10800), $sends('~^send -P -v -t tok:tank/new(/sub)?@auto-20270115100000$~'), $tokens('backup/tank/cut'), array_values(array_diff(array('backup/tank/cut@auto-20270115100000', 'backup/tank/cut/sub@auto-20270115100000'), $marks()))), 'the next run carries each on from its token, and both copies land'); +same(array(), $pinned('tank/cut'), 'and lets go of their holds'); // Into a dataset whose copy is not made yet: an empty holder takes its place first, as zfs rename needs the parent. $pool('datasets', array_merge($read('pool/datasets'), array("tank/deep\tfilesystem"))); $reset(); @@ -425,9 +431,9 @@ function same($expected, $actual, $message) { putenv('ZFSM_TEST_COVER_FAIL'); same(array(array($longer('tank/sz') => 0, $longer('tank/ro') => 0, $longer('tank/lg') => 0), array($longer('tank/un') => 'Nothing was renamed: the names of '.$longer('tank/un').' and its copy would be too long for the bookmark a copy keeps; pick a shorter name')), array($renamed, $unread), 'one a run leaves out may take a long name; one not read may not'); -// One whose read-only or legacy mount may come from above it is checked under its new parent; one that sets its own is not. +// One that inherits a read-only or legacy mount from above it is checked under its new parent; one that sets its own is not. $add(array('tank/rp', 'tank/rp/c', 'tank/lp', 'tank/lp/c', 'tank/rq', 'tank/rq/c')); -$pool('props', array_merge($read('pool/props'), array("tank/rp\treadonly\ton", "tank/rp/c\treadonly\ton", "tank/lp\tmountpoint\tlegacy", "tank/lp/c\tmountpoint\tlegacy", "tank/rq/c\treadonly\ton"))); +$pool('props', array_merge($read('pool/props'), array("tank/rp\treadonly\ton", "tank/lp\tmountpoint\tlegacy", "tank/rq/c\treadonly\ton"))); $moved = array(); foreach (array('tank/rp/c' => 'tank/rpc', 'tank/lp/c' => 'tank/lpc', 'tank/rq/c' => 'tank/rqc') as $zdataset => $to) { $answer = renameDataset($zdataset, $longer($to), 0, $cfg); @@ -436,10 +442,20 @@ function same($expected, $actual, $message) { $too = function($zdataset) { return 'Nothing was renamed: the names of '.$zdataset.' and its copy would be too long for the bookmark a copy keeps; pick a shorter name'; }; same(array($longer('tank/rpc') => $too($longer('tank/rpc')), $longer('tank/lpc') => $too($longer('tank/lpc')), $longer('tank/rqc') => 0), $moved, 'one moved out from under a read-only or legacy parent is checked; one read-only itself is not'); -putenv('ZFSM_TEST_COVER_FAIL=tank'); -$unsure = renameDataset($longer('tank/rqc'), $longer('tank/rqd'), 0, $cfg)['failed']; +// Under a legacy or read-only parent, or inheriting that from a dataset that moves with it, no run copies it; with the parent or the +// sources not read, it is checked. +$add(array('tank/arch', 'tank/frz', 'tank/d1', 'tank/d2', 'tank/d3', 'tank/lx', 'tank/lx/c', 'tank/lq', 'tank/lq/c')); +$pool('props', array_merge($read('pool/props'), array("tank/arch\tmountpoint\tlegacy", "tank/frz\treadonly\ton", "tank/lx\tmountpoint\tlegacy", "tank/lq\tmountpoint\tlegacy"))); +$into = array(renameDataset('tank/d1', $longer('tank/arch/d1'), 0, $cfg)['succeeded'], renameDataset('tank/d2', $longer('tank/frz/d2'), 0, $cfg)['succeeded'], + renameDataset('tank/lx', $longer('tank/lxx'), 0, $cfg)['succeeded']); +putenv('ZFSM_TEST_COVER_FAIL=tank/arch'); +$into[] = renameDataset('tank/d3', $longer('tank/arch/d3'), 0, $cfg)['failed']; putenv('ZFSM_TEST_COVER_FAIL'); -same(array($longer('tank/rqd') => $too($longer('tank/rqd'))), $unsure, 'one whose parents could not be read is checked'); +putenv('ZFSM_TEST_SOURCE_FAIL=1'); +$into[] = renameDataset('tank/lq/c', $longer('tank/lqc'), 0, $cfg)['failed']; +putenv('ZFSM_TEST_SOURCE_FAIL'); +same(array(array($longer('tank/arch/d1') => 0), array($longer('tank/frz/d2') => 0), array($longer('tank/lxx') => 0), array($longer('tank/arch/d3') => $too($longer('tank/arch/d3'))), + array($longer('tank/lqc') => $too($longer('tank/lqc')))), $into, 'one left out once moved may take a long name; with the parent or the sources not read it is checked'); // Its own copies go with it, so the room is that under their Copy Under. $far = 'backup/'.str_repeat('c', 20); $kin = str_repeat('n', 50); diff --git a/zfs.master/include/ZFSMScheduleCopies.php b/zfs.master/include/ZFSMScheduleCopies.php index 1a95d91c..7b15d4c3 100644 --- a/zfs.master/include/ZFSMScheduleCopies.php +++ b/zfs.master/include/ZFSMScheduleCopies.php @@ -146,7 +146,7 @@ function zfsmScheduleCopiesMade($job, $jobs, $cfg, $now, $wait) { $problems = $waiting = array(); foreach ($datasets as $zdataset) { $zdest = zfsmScheduleCopiesDest($job, $zdataset); - $place = zfsmScheduleCopiesPlace($job['dest'], $zdest, $coming, $here); + $place = zfsmScheduleCopiesPlace($job['dest'], $zdest, $coming, $here, $cfg); if ($place === null) $waiting[] = $zdataset; if ($place !== '') { if ($place !== null) $problems[] = $place; @@ -176,8 +176,6 @@ function zfsmScheduleCopiesComing($job, $jobs, $listed) { $coming = array(); foreach (zfsmScheduleListed($listed)[0] as $zdataset => $row) { $by = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); - // One whose snapshot names were not read may be due too. - $row[2] = $row[2] ?? false; if (($by['dest'] ?? '') === $job['dest'] && zfsmScheduleLeftOut($by, $jobs, $zdataset, $row, $listed['name']) === '') $coming[zfsmScheduleCopiesDest($by, $zdataset)] = true; } foreach (zfsmScheduleCopiesRepeatsTo($jobs) as $zdest => $from) $coming[$zdest] = true; @@ -222,9 +220,9 @@ function zfsmScheduleCopiesLevels($dest, $zdest) { return $levels; } -// The datasets above $zdest, a copy under $dest: '' when each is there or made a holder, null to wait where a copy is $coming, else -// why a holder could not be made, as for a dataset renamed or destroyed since a run listed it. $here gains each holder. -function zfsmScheduleCopiesPlace($dest, $zdest, $coming, &$here) { +// The datasets above $zdest, a copy under $dest: '' when each is there or made a holder, null to wait where a copy is $coming or (a +// run's, given $cfg) due by the schedules saved now, else why a holder could not be made, as for a dataset gone since. $here gains each. +function zfsmScheduleCopiesPlace($dest, $zdest, $coming, &$here, $cfg = null) { $zdataset = substr($zdest, strlen($dest)); $missing = null; foreach (zfsmScheduleCopiesLevels($dest, $zdest) as $level) { @@ -232,14 +230,23 @@ function zfsmScheduleCopiesPlace($dest, $zdest, $coming, &$here) { if (isset($coming[$level])) return null; $missing = $missing ?? zfsmDatasetMissing($zdataset); if ($missing !== false) return ($missing ? $zdataset.' is gone' : 'whether '.$zdataset.' is still there could not be read').', so no dataset was made to hold its copy'; - $why = zfsmReplicaHolderMake($level); - // Another run may have made it since this one listed. - if ($why !== '' && !zfsmDatasetExists($level, 'filesystem')) return $why; + // Read again, as a run takes hours: another may have made it since, or a schedule been saved to copy there. + if (!zfsmDatasetExists($level, 'filesystem')) { + if ($cfg !== null && zfsmScheduleCopiesDueNow($dest, $level, $cfg)) return null; + $why = zfsmReplicaHolderMake($level); + if ($why !== '') return $why; + } $here[$level] = true; } return ''; } +// Whether the schedules saved now copy a dataset to $level under $dest (zfsmScheduleCopiesTo), or that could not be read. +function zfsmScheduleCopiesDueNow($dest, $level, $cfg) { + $read = zfsmScheduleRead(); + return $read === null || zfsmScheduleCopiesTo($dest, $level, $read['valid'], $cfg) !== array(); +} + // Why renaming $from to $to ($names: it and those below it) would leave a copy a schedule makes of one no room for the bookmark it // keeps (zfsmReplicaBookmarkFits), where there was; '' when none. function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { @@ -249,14 +256,14 @@ function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { $snapshot = zfsmNewSnapshotName($cfg, time()); // One a run leaves out (zfsmScheduleSkipped), there and once moved, needs no room; one whose row could not be read is checked. Read // once a name is too long. - $rows = $parents = null; - $skipped = function($zdataset, $name) use (&$rows, &$parents, $cfg, $from, $to) { + $rows = $moving = null; + $skipped = function($zdataset, $name) use (&$rows, &$moving, $cfg, $from, $to) { if ($rows === null) { $rows = zfsmScheduleListed(zfsmScheduleCoverRead($cfg, ZFSM_SCHEDULE_COVER_WAIT, array('-r', $from)))[0]; - $parents = zfsmScheduleCopiesParents($from, $to); + $moving = zfsmScheduleCopiesMoving($from, $to); } if (!isset($rows[$name])) return false; - return zfsmScheduleSkipped($zdataset, $zdataset === $name ? $rows[$name] : zfsmScheduleCopiesMovedRow($rows[$name], ...$parents)) !== ''; + return zfsmScheduleSkipped($zdataset, $zdataset === $name ? $rows[$name] : zfsmScheduleCopiesMovedRow($rows[$name], $name, $from, ...$moving)) !== ''; }; $fits = function($zdataset, $jobs, $name) use ($snapshot, $skipped) { $job = zfsmScheduleFollowed($zdataset, $jobs, 'zfsmScheduleCopiesUnder'); @@ -272,19 +279,29 @@ function zfsmScheduleCopiesFit($names, $from, $to, $cfg) { return ''; } -// The rows (zfsmScheduleListed) of the datasets above $from and $to, each null when not read. -function zfsmScheduleCopiesParents($from, $to) { - $up = array_map(function($zdataset) { return substr($zdataset, 0, (int)strrpos($zdataset, '/')); }, array($from, $to)); - $listed = runProcess('zfs', array_merge(ZFSM_SCHEDULE_COVERED, $up)); - $rows = ($listed['code'] ?? 1) === 0 ? zfsmScheduleListed(array('output' => $listed['output']))[0] : array(); - return array($rows[$up[0]] ?? null, $rows[$up[1]] ?? null); +// The row (zfsmScheduleListed) of $to's parent, and where readonly and mountpoint of $from and each below it come from, by name; each +// null when not read. +function zfsmScheduleCopiesMoving($from, $to) { + $parent = substr($to, 0, (int)strrpos($to, '/')); + $listed = runProcess('zfs', array_merge(ZFSM_SCHEDULE_COVERED, array($parent))); + $row = ($listed['code'] ?? 1) === 0 ? zfsmScheduleListed(array('output' => $listed['output']))[0][$parent] ?? null : null; + $read = runProcess('zfs', array('get', '-H', '-o', 'name,property,source', '-t', 'filesystem,volume', '-r', 'readonly,mountpoint', $from)); + if (($read['code'] ?? 1) !== 0) return array($row, null); + $sources = array(); + foreach (zfsmLines($read['output']) as $line) { + list($name, $property, $source) = explode("\t", $line, 3) + array('', '', ''); + $sources[$name][$property === 'readonly' ? 0 : 1] = $source; + } + return array($row, $sources); } -// $row of a dataset at or below one moved from under $old to under $new (zfsmScheduleCopiesParents): a read-only or legacy mount it -// may take from $old takes $new's. -function zfsmScheduleCopiesMovedRow($row, $old, $new) { - foreach (array('on', 'legacy') as $i => $skip) { - if ($row[$i] === $skip && ($old[$i] ?? $skip) === $skip) $row[$i] = $new[$i] ?? ''; +// $row of $name, at or below $from, once moved under $parent (zfsmScheduleCopiesMoving): a value it inherits from above $from, or +// has by default, takes $parent's; one whose source or parent was not read gives no reason to leave it out. +function zfsmScheduleCopiesMovedRow($row, $name, $from, $parent, $sources) { + foreach (array(0, 1) as $i) { + $source = $sources[$name][$i] ?? null; + $above = $source === 'default' || (strpos((string)$source, 'inherited from ') === 0 && !zfsmUnderAny(substr($source, 15), array($from))); + if ($source === null || $above) $row[$i] = $source === null ? '' : ($parent[$i] ?? ''); } return $row; } @@ -328,9 +345,8 @@ function zfsmScheduleCopiesMove($from, $to, $before, $after, $cfg) { return $stayed; } -// The copy of $from under $job's dest, with its bookmarks, renamed to $to's when it is a copy of $to or a holder and a schedule in -// $after still copies $to or one below it there; '' when it moved, is not there, or is not read where none copies it, else what -// went wrong. +// The copy of $from under $job's dest, with its bookmarks, renamed to $to's when it is $to's or a holder and $after still copies there; +// '' when it moved or is not there (or not read where none copies it), else what went wrong. function zfsmScheduleCopiesMoveOne($job, $from, $to, $after, $cfg) { $old = zfsmScheduleCopiesDest($job, $from); $new = zfsmScheduleCopiesDest($job, $to); @@ -479,9 +495,8 @@ function zfsmScheduleCopiesBlocked($dest, $zdest, $here, $jobs, $cfg) { return ''; } -// The datasets $jobs copy to $level under $dest, as zfsmScheduleCopiesComing reads a run's: the one of that name (above the renamed -// one, so there) when a schedule's run takes it there (zfsmScheduleLeftOut), and each a repeat set in Replicate sends there; null -// when whether a run takes it could not be read. +// The datasets $jobs copy to $level under $dest, as zfsmScheduleCopiesComing reads a run's: its own where a run takes it there, and +// each a repeat set in Replicate sends there; null when whether a run takes it could not be read. function zfsmScheduleCopiesTo($dest, $level, $jobs, $cfg) { $zdataset = substr($level, strlen($dest)); $repeats = zfsmScheduleCopiesRepeatsTo($jobs)[$level] ?? array(); From 996c08b498def8ea803bcdf9f7c3a7aa9684ef32 Mon Sep 17 00:00:00 2001 From: chodeus <190988615+chodeus@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:11:42 +0800 Subject: [PATCH 24/24] style: two comments cut to two lines --- tests/fixtures/bin/zfs | 5 ++--- zfs.master/include/ZFSMSchedule.php | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/fixtures/bin/zfs b/tests/fixtures/bin/zfs index c7c5f1db..acaee31a 100644 --- a/tests/fixtures/bin/zfs +++ b/tests/fixtures/bin/zfs @@ -251,9 +251,8 @@ if [ -n "${ZFSM_TEST_REPL_DIR:-}" ] && ! { [ "$1" = destroy ] && contains_arg -n awk -F'\t' -v n="$last_arg" -v p="$2" '!($1 == n && $2 == p)' "$R/props" > "$R/props.new" && mv "$R/props.new" "$R/props" exit 0 ;; rename) - # Moves it and all below it, with their snapshots, bookmarks, properties and holds; ZFSM_TEST_RENAME_FAIL names one that is busy, - # ZFSM_TEST_RENAME_UNMOUNTED one that moves, then fails to mount there. - # ZFSM_TEST_LOCK_PROBE: a lock file whose state at each rename goes to probe.log, held or free, as ZFSM_TEST_PHP finds it. + # Moves it and all below it, with snapshots, bookmarks, properties and holds. ZFSM_TEST_RENAME_FAIL names one busy, ZFSM_TEST_RENAME_UNMOUNTED + # one that moves then fails to mount; ZFSM_TEST_LOCK_PROBE is a lock file whose state at each rename (held or free) goes to probe.log. [ -z "${ZFSM_TEST_LOCK_PROBE:-}" ] || "$ZFSM_TEST_PHP" -r 'echo flock(fopen(getenv("ZFSM_TEST_LOCK_PROBE"), "c"), LOCK_EX | LOCK_NB) ? "free\n" : "held\n";' >> "$R/probe.log" eval "from=\${$(($# - 1))}" exists "$from" || { printf "cannot open '%s': dataset does not exist\n" "$from" >&2; exit 1; } diff --git a/zfs.master/include/ZFSMSchedule.php b/zfs.master/include/ZFSMSchedule.php index 07604669..cd460abf 100644 --- a/zfs.master/include/ZFSMSchedule.php +++ b/zfs.master/include/ZFSMSchedule.php @@ -303,9 +303,8 @@ function zfsmScheduleSame($jobs, $job) { return ''; } -// As zfsmScheduleChange answers: removed once 'saved', even if the crontab could not be rebuilt after; not when the schedule above -// that its datasets then follow clashes with one as saved (zfsmScheduleNewClash, Snapshots Pattern from $cfg), unless its dataset is -// gone (zfsmDatasetGone), so none follows it. +// As zfsmScheduleChange answers: removed once 'saved', even if the crontab could not be rebuilt; not when the schedule its datasets then +// follow clashes with one saved (zfsmScheduleNewClash, Snapshots Pattern from $cfg), unless its dataset is gone (zfsmDatasetGone). function zfsmScheduleRemove($id, $cfg) { if (!zfsmScheduleId($id)) return array('saved' => false, 'why' => 'No such schedule'); $changed = zfsmScheduleChange(function($jobs) use ($id, $cfg) {