diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 24006ece..65ac2287 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -206,6 +206,20 @@ reviews: GraphQL checks in schema.py; flag a second copy of either. - TESTS use synthetic versions, shas and handles only; a value captured from a live system in a fixture is a finding. + - path: ".github/unraid-compat/**" + instructions: | + The Unraid compatibility matrix; CI only, nothing here ships. + - install_test.sh puts the .plg through each Unraid version's own + plugin installer, hooks included, in a fresh Docker container with + no network; it must refuse to run anywhere else. Only Slackware's + upgradepkg and removepkg are stubbed, keeping their file-list + records: flag any other stand-in for Unraid's code. + - matrix_check.py keeps ci.yml's Unraid rows at one per minor from the + .plg's min to the newest release; Renovate only moves rows. + - Each step asserts what the user is left with (live files, the + registered manifest, saved folders, Unraid's orders) right after it, + against values written by hand. A failed step fails the job. + - Fixtures hold synthetic names and ids only. - path: ".github/scripts/**" instructions: | CI tests and helpers; nothing here ships in the plugin. @@ -224,6 +238,9 @@ reviews: failed read or download as "nothing to report". - package.json here is test-only (jquery, jsdom), pinned to exact versions. + - php_tests.sh is the one list of PHP checks for Lint + build and the + PHP matrix: php -l on every .php and .page file, then every + *_test.php here; a deprecation fails like an error. - path: "**" instructions: |- Applies to every file, whatever the language: diff --git a/.github/scripts/important_check.py b/.github/scripts/important_check.py new file mode 100644 index 00000000..489c0fd1 --- /dev/null +++ b/.github/scripts/important_check.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Each !important in the plugin's own CSS, JS, PHP and .page files needs a `!important: ` comment.""" +import pathlib +import re +import sys + +import vendor + +REASON = re.compile(r'!important:\s*\w', re.I) +KINDS = {'.css': 'css', '.js': 'js', '.php': 'php', '.page': 'page'} +# A `/` after one of these, or first on its line, opens a JS regex literal rather than dividing; not `<`, as `

` has it +REGEX_AFTER = set('(,=:[!&|?{};+-*%>~^') + + +def _blank(code, text, i, j): + code[i:j] = [ch if ch == '\n' else ' ' for ch in text[i:j]] + + +def _regex_end(text, i): + """End of the JS regex literal opening at `i`, or None when the `/` divides.""" + before = text[:i].rstrip(' \t') + if before and not before.endswith('\n') and before[-1] not in REGEX_AFTER and not re.search(r'\b(return|typeof)$', before): + return None + j, in_class = i + 1, False + while j < len(text) and text[j] != '\n': + if text[j] == '\\': + j += 1 + elif text[j] == '[': + in_class = True + elif text[j] == ']': + in_class = False + elif text[j] == '/' and not in_class: + return j + 1 + j += 1 + return None + + +def _in_script(spans, i): + return any(start <= i < end for start, end in spans) + + +def scan(text, kind): + """(code, comments): the text with comments, CSS strings and JS regexes blanked, and [(start, end, comment text)].""" + code, comments, i, n = list(text), [], 0, len(text) + script = kind != 'css' + # A .page holds CSS and HTML too: only its ', text, re.I | re.S)] if kind == 'page' else [] + slashes = [(0, n)] if kind in ('js', 'php') else js + [m.span(1) for m in re.finditer(r'<\?(.*?)(?:\?>|\Z)', text, re.S)] if kind == 'page' else [] + while i < n: + c, two = text[i], text[i:i + 2] + html = kind == 'page' and text.startswith('' if html else '\n' + found = text.find(close, i + 2) + end = n if found == -1 else found + (0 if close == '\n' else len(close)) + comments.append((i, end, text[i:end])) + _blank(code, text, i, end) + i = end + elif c == '/' and _in_script(js, i) and (end := _regex_end(text, i)): + _blank(code, text, i, end) + i = end + elif c in '"\'' or (c == '`' and script): + j = i + 1 + while j < n and text[j] != c and text[j] != '\n': + j += 2 if text[j] == '\\' else 1 + # A script's string is where inline styles live: keep it as code; a CSS string (content: "...") is text + if kind == 'css': + _blank(code, text, i, j + 1) + i = j + 1 + else: + i += 1 + return ''.join(code), comments + + +def _line(text, at): + return text.count('\n', 0, at) + + +def _own_line(text, start): + return text[text.rfind('\n', 0, start) + 1:start].strip() == '' + + +def _reason(comments, text, line, alone): + """A reason comment ending on `line`; with `alone`, only one with no code before it on its line.""" + return any(REASON.search(body) and _line(text, end - 1) == line and (not alone or _own_line(text, start)) + for start, end, body in comments) + + +def _rule_start(code, at): + """Where the selector of the rule holding the declaration at `at` begins, past any nested rules; None outside a rule.""" + depth, opened = 0, at - 1 + while opened >= 0 and not (code[opened] == '{' and depth == 0): + depth += (code[opened] == '}') - (code[opened] == '{') + opened -= 1 + if opened < 0: + return None + start = max(code.rfind(ch, 0, opened) for ch in '{};') + 1 + while start < opened and code[start].isspace(): + start += 1 + return start + + +def problems_in(path, text, kind): + code, comments = scan(text, kind) + out = [] + for m in re.finditer(r'!\s*important\b', code, re.I): + line = _line(text, m.start()) + ok = _reason(comments, text, line, alone=False) or _reason(comments, text, line - 1, alone=True) + start = _rule_start(code, m.start()) if kind == 'css' else None + if not ok and start is not None: + ok = _reason(comments, text, _line(text, start) - 1, alone=True) or _reason(comments, text, _line(text, start), alone=True) + if not ok: + out.append(f'{path}:{line + 1}: !important without a "!important: " comment on its line, the line above, or above its rule') + return out + + +def problems(repo): + repo = repo.resolve() + manifest, root = vendor.plugin_root(repo) + bundled = {vendor.destination(manifest, root, dest) for lib in manifest['libraries'] for dest in lib['files'].values()} + files = [p for p in sorted(root.rglob('*')) if p.suffix in KINDS and p.is_file() and p.resolve() not in bundled] + if not files: + return [f'{manifest["root"]}: no CSS, JS, PHP or .page files to check'] + return [problem for p in files for problem in problems_in(p.relative_to(repo), p.read_text(encoding='utf-8'), KINDS[p.suffix])] + + +if __name__ == '__main__': + found = problems(pathlib.Path(__file__).resolve().parents[2]) + print('\n'.join(found) or 'Every !important has its reason') + sys.exit(1 if found else 0) diff --git a/.github/scripts/important_check_test.py b/.github/scripts/important_check_test.py new file mode 100644 index 00000000..ad451f49 --- /dev/null +++ b/.github/scripts/important_check_test.py @@ -0,0 +1,98 @@ +import json +import pathlib +import tempfile +import unittest + +import important_check + +ROOT = 'src/plugin' + + +class ImportantCheckTest(unittest.TestCase): + def check(self, files, bundled=()): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + repo = pathlib.Path(tmp.name) + (repo / 'vendor.json').write_text(json.dumps({'root': ROOT, 'libraries': [{'files': {f'x/{b}': b for b in bundled}}]}), encoding='utf-8') + for name, text in files.items(): + (repo / ROOT / name).parent.mkdir(parents=True, exist_ok=True) + (repo / ROOT / name).write_text(text, encoding='utf-8') + return important_check.problems(repo) + + def test_a_reason_above_the_rule_or_on_the_line_passes(self): + sheet = ('.a { top: 0 }\n\n/* !important: Unraid inlines this padding */\n#t .x,\n#t .y {\n padding-left: 20px !important;\n}\n' + '.b { margin: 0 !important; /* !important: a theme sets it inline */ }\n') + self.assertEqual(self.check({'styles/a.css': sheet}), []) + + def test_an_important_without_its_reason_is_named_with_its_line(self): + sheet = '/* spacing */\n.a {\n top: 0;\n left: 1px!important;\n}\n' + self.assertEqual(self.check({'styles/a.css': sheet}), + [f'{ROOT}/styles/a.css:4: !important without a "!important: " comment on its line, the line above, or above its rule']) + + def test_a_reason_belongs_to_the_rule_under_it_only(self): + sheet = '/* !important: first */\n.a { top: 0 !important }\n.b { top: 0 !important }\n' + (problem,) = self.check({'styles/a.css': sheet}) + self.assertIn('a.css:3:', problem) + + def test_a_reason_at_the_end_of_a_line_covers_that_line_only(self): + rules = '.a { top: 0 !important } /* !important: r */\n.b { top: 0 !important }\n' + decls = '.c {\n top: 0 !important; /* !important: r */\n left: 0 !important;\n}\n' + self.assertEqual([p.split(': !')[0] for p in self.check({'styles/a.css': rules, 'styles/b.css': decls})], + [f'{ROOT}/styles/a.css:2', f'{ROOT}/styles/b.css:3']) + + def test_a_nested_rule_does_not_move_a_declaration_to_its_selector(self): + outer = '/* !important: r */\n.a {\n .b { color: red }\n top: 0 !important;\n}\n' + inner = '.a {\n /* !important: r */\n .b { color: red }\n top: 0 !important;\n}\n' + self.assertEqual(self.check({'styles/a.css': outer}), []) + (problem,) = self.check({'styles/b.css': inner}) + self.assertIn('b.css:4:', problem) + + def test_a_quote_in_a_regex_literal_or_a_php_hash_comment_is_read_as_code_does(self): + js = "s = s.replace(/'/g, ''); el.style.cssText = 'x:1 !important'; // !important: r\n" + php = "See https://example.test

\n\n\n'}) + self.assertEqual([p.split(': !')[0] for p in found], [f'{ROOT}/Folder.page:1', f'{ROOT}/styles/a.css:1']) + + def test_a_url_in_page_css_is_no_regex_but_one_in_its_script_is(self): + page = ('\n' + "\n") + (problem,) = self.check({'Folder.page': page}) + self.assertIn('Folder.page:1:', problem) + + def test_a_protocol_relative_url_in_page_css_is_no_comment_but_one_in_its_php_is(self): + page = '\n"; ?>\n' + (problem,) = self.check({'Folder.page': page}) + self.assertIn('Folder.page:1:', problem) + + def test_no_files_to_check_fails_closed(self): + self.assertEqual(self.check({'README.md': 'x'}), [f'{ROOT}: no CSS, JS, PHP or .page files to check']) + + def test_a_reason_comment_must_give_a_reason(self): + self.assertEqual(len(self.check({'styles/a.css': '/* !important: */\n.a { top: 0 !important }\n'})), 1) + + def test_a_comment_or_a_content_string_that_mentions_it_is_not_a_use(self): + sheet = '/* never use !important here */\n.a::after { content: "!important"; }\n' + self.assertEqual(self.check({'styles/a.css': sheet}), []) + + def test_a_script_needs_the_reason_on_its_line_or_the_line_above(self): + good = "// beats swal's !important rules\n// !important: no stylesheet may show a row while it is measured\nrow.style.cssText += ';visibility:hidden !important;';\n" + bad = "// hide the row\nrow.style.cssText += ';visibility:hidden !important;';\n" + self.assertEqual(self.check({'scripts/a.js': good}), []) + (problem,) = self.check({'scripts/b.js': bad}) + self.assertIn('scripts/b.js:2:', problem) + + def test_page_and_php_files_are_checked_too(self): + found = self.check({'Folder.page': 'Menu="x"\n---\n\n', 'server/a.php': " "$log" +printf 'error_reporting=-1\nlog_errors=1\nerror_log=%s\n' "$log" > "$ini/zz-fv3-tests.ini" +# An empty entry keeps PHP's own scan folder; this file loads after it, in every child process too +export PHP_INI_SCAN_DIR="${PHP_INI_SCAN_DIR:-}:$ini" +php -v | head -1 +php -r 'exit((error_reporting() & E_ALL) === E_ALL && ini_get("log_errors") ? 0 : 1);' || { echo 'FAILED: the error-log settings did not load'; exit 1; } + +deprecated() { # deprecated : fails the run when logged a deprecation + local seen; seen=$(grep -c 'Deprecated:' "$log") + if [ "$seen" -gt "${logged:-0}" ]; then + echo "FAILED: $1 ran into a deprecation:" + grep 'Deprecated:' "$log" | tail -n "$((seen - ${logged:-0}))" + status=1 + fi + logged=$seen +} + +while IFS= read -r -d '' f; do + php -l "$f" >/dev/null 2>&1 || { php -l "$f"; status=1; } +done < <(find src \( -name '*.php' -o -name '*.page' \) -print0) +deprecated 'php -l' + +for t in .github/scripts/*_test.php; do + echo "== $t" + php -d display_errors=stderr "$t" || status=1 + deprecated "$t" +done +exit $status diff --git a/.github/scripts/vendor.py b/.github/scripts/vendor.py index ef2e1f91..0437defb 100755 --- a/.github/scripts/vendor.py +++ b/.github/scripts/vendor.py @@ -95,12 +95,25 @@ def lf(data): return data if b'\0' in data[:8000] else data.replace(b'\r\n', b'\n') +def plugin_root(repo=ROOT): + """(vendor.json, the plugin folder its root names).""" + manifest = json.loads((repo / 'vendor.json').read_text()) + return manifest, (repo / manifest['root']).resolve() + + +def destination(manifest, root, rel): + """Where a bundled file lands; vendor.json may not point outside the plugin folder.""" + dest = (root / rel).resolve() + if not dest.is_relative_to(root): + raise SystemExit(f'vendor.json: {rel} is outside {manifest["root"]}') + return dest + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('mode', choices=('check', 'sync')) mode = parser.parse_args().mode - manifest = json.loads((ROOT / 'vendor.json').read_text()) - root = (ROOT / manifest['root']).resolve() + manifest, root = plugin_root() stale = [] for lib in manifest['libraries']: if 'npm' in lib: @@ -110,9 +123,7 @@ def main(): else: raise SystemExit(f'vendor.json: a library needs an npm or github source: {lib}') for src, data in files.items(): - dest = (root / lib['files'][src]).resolve() - if not dest.is_relative_to(root): - raise SystemExit(f'vendor.json: {lib["files"][src]} is outside {manifest["root"]}') + dest = destination(manifest, root, lib['files'][src]) data = lf(data) if dest.is_file() and dest.read_bytes() == data: continue diff --git a/.github/unraid-compat/Dockerfile b/.github/unraid-compat/Dockerfile new file mode 100644 index 00000000..2f8f421f --- /dev/null +++ b/.github/unraid-compat/Dockerfile @@ -0,0 +1,5 @@ +# The PHP an Unraid version ships, plus what Unraid has and this image lacks: gawk (the .plg's remove script uses RT), xz +# (the package format) and wget (the installer's downloader, so a download fails on the network, not a missing tool). +ARG PHP +FROM php:${PHP}-cli +RUN apt-get update && apt-get install -y --no-install-recommends gawk wget xz-utils && rm -rf /var/lib/apt/lists/* diff --git a/.github/unraid-compat/install_test.sh b/.github/unraid-compat/install_test.sh new file mode 100755 index 00000000..2612bb54 --- /dev/null +++ b/.github/unraid-compat/install_test.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# FolderView3 through a real Unraid webGUI's plugin installer: install, upgrade over the last stable, remove, failures, min. +# Disposable container only, as root, with no network. Usage: install_test.sh +set -u +# This replaces /usr/bin/php, package tools and plugin files, so it runs only in a fresh Docker container, never on a host. +if [ ! -f /.dockerenv ] || [ -e /etc/unraid-version ] || [ -d /boot/config ]; then echo 'Refusing to run outside a fresh Docker container' >&2; exit 2; fi +version=$2; repo=$(pwd) +fail() { echo "FAILED [Unraid $version]: $*"; exit 1; } +web=$(cd "$1" 2>/dev/null && pwd) && [ -f "$web/emhttp/plugins/dynamix.plugin.manager/scripts/plugin" ] || fail "no Unraid plugin installer under $1" +stable=$(cd "$3" 2>/dev/null && pwd) && [ -f "$stable/pkg_build.sh" ] || fail "no last-stable checkout with pkg_build.sh at $3" +awk --version 2>/dev/null | grep -q 'GNU Awk' || fail 'awk is not gawk, as it is on Unraid: the remove script needs RT' + +name=folder.view3 +docroot=/usr/local/emhttp +flash=/boot/config/plugins/$name +live=$docroot/plugins/$name +manifest=/boot/config/plugins/$name.plg +mkdir -p "$docroot/plugins" /boot/config/plugins-error /var/log/plugins /var/log/packages /var/local/emhttp +cp -R "$web/emhttp/plugins/dynamix" "$web/emhttp/plugins/dynamix.plugin.manager" "$docroot/plugins/" +ln -sfn "$docroot/plugins/dynamix" "$docroot/webGui" +printf 'csrf_token="compat-test-token"\n' > /var/local/emhttp/var.ini +# Unraid's pre- and post-install hooks run as #!/usr/bin/php and keep their state under /tmp/plugins +mkdir -p /tmp/plugins +ln -sf "$(command -v php)" /usr/bin/php +plugin() { php "$docroot/plugins/dynamix.plugin.manager/scripts/plugin" "$@"; } + +# Slackware's package tools, as they work: the record lists the package's files, and removepkg takes those and the folders +# they leave empty. Everything else is Unraid's own code. +cat > /sbin/upgradepkg <<'STUB' +#!/bin/bash +for pkg; do :; done +tar -xJf "$pkg" -C / || exit 1 +tar -tJf "$pkg" > "/var/log/packages/$(basename "$pkg" .txz)" +STUB +cat > /sbin/removepkg <<'STUB' +#!/bin/bash +record=/var/log/packages/$1 +[ -f "$record" ] || exit 1 +grep -v '/$' "$record" | while IFS= read -r f; do rm -f "/$f"; done +grep '/$' "$record" | sort -r | while IFS= read -r d; do rmdir "/$d" 2>/dev/null; done +rm -f "$record" +STUB +chmod +x /sbin/upgradepkg /sbin/removepkg + +work=$(mktemp -d) +build() { # build : the .plg and package as the release workflow makes them + local dir="$work/$2"; mkdir -p "$dir"; cp -R "$1/src" "$1/$name.plg" "$1/pkg_build.sh" "$dir/" + (cd "$dir" && bash pkg_build.sh --version "$2" --branch "$3" --out "$dir/dist" >/dev/null) || fail "package build of $2" +} +old=2026.01.01; new=2099.01.01 +build "$stable" $old main; build "$repo" $new beta +min=$(grep -o ']*min="[^"]*"' "$work/$new/$name.plg" | sed 's/.*min="//; s/"$//') +[ -n "$min" ] || fail 'the .plg names no min version' + +set_version() { printf 'version="%s"\n' "$1" > /etc/unraid-version; } +reset() { + rm -rf "$flash" "$live" "$manifest" /var/log/plugins/$name.plg /var/log/packages/$name-* /boot/config/plugins/dockerMan /boot/config/plugins/dynamix.vm.manager + mkdir -p "$flash" +} +install() { # install : the .plg with its package already on the flash, as a boot or a cached download has it + cp "$work/$1/dist/$name-$1-x86_64-1.txz" "$flash/" + plugin install "$work/$1/$name.plg" > "$work/$2.log" 2>&1 || { cat "$work/$2.log"; fail "$2: install of $1 refused"; } +} +installed() { # installed : its files are live, its package is recorded and its manifest registered + local pkg=$name-$1-x86_64-1 x=$work/unpacked + rm -rf "$x" && mkdir "$x" && tar -xJf "$work/$1/dist/$pkg.txz" -C "$x" || fail "$2: unpack $1" + local differ; differ=$(cd "$x" && find . -type f | while read -r f; do cmp -s "$f" "/$f" || echo "${f#.}"; done) + [ -z "$differ" ] || fail "$2: files not as $1 ships them: $(echo $differ | head -c 300)" + [ -e "/var/log/packages/$pkg" ] || fail "$2: no package record for $1" + [ -L /var/log/plugins/$name.plg ] && grep -q "ENTITY version \"$1\"" "$manifest" || fail "$2: $1 is not the registered manifest" +} +saved_config() { # what a user has saved: folders, settings, a theme, and Unraid's own orders holding folder entries + mkdir -p "$flash/styles" "$flash/scripts" /boot/config/plugins/dockerMan /boot/config/plugins/dynamix.vm.manager + printf '{"AbCdEfGhIjKlMnOpQrSt":{"name":"Media","icon":"","regex":"","containers":["alpha"],"settings":{"preview":1},"actions":[]}}' > "$flash/docker.json" + printf '{"ZyXwVuTsRqPoNmLkJiHg":{"name":"Lab","icon":"","regex":"","containers":["Debian 12"],"settings":{"preview":1},"actions":[]}}' > "$flash/vm.json" + printf '{"docker":{"preview":1}}' > "$flash/settings.json" + printf '.folder-appname { color: #abc; }\n' > "$flash/styles/custom.css" + # folder-sync and folder-nightlybackups are containers: too short, or no capital as a folder id has + printf '0="folder-AbCdEfGhIjKlMnOpQrSt"\n1="alpha"\n2="folder-sync"\n3="folder-nightlybackups"\n4="beta"\n' > /boot/config/plugins/dockerMan/userprefs.cfg + printf '0="folder-ZyXwVuTsRqPoNmLkJiHg"\r\n1="Debian 12"\r\n2="Windows"' > /boot/config/plugins/dynamix.vm.manager/userprefs.cfg +} +config_sums() { (cd /boot/config/plugins && find $name dockerMan dynamix.vm.manager -type f ! -name "$name-*.txz" ! -path "$name/version" -print0 | sort -z | xargs -0 md5sum); } +boot() { # boot : what a reboot does: RAM starts empty, and the registered manifest installs again from the flash alone + rm -rf "$live" /var/log/packages/$name-* /var/log/plugins/$name.plg + plugin install "$manifest" > "$work/$1.log" 2>&1 || { cat "$work/$1.log"; fail "$1: reinstall at boot refused"; } +} +set_version "$version" + +# A fresh install +reset; install $new fresh; installed $new fresh +[ "$(cat "$flash/version")" = $new ] || fail 'fresh: version file not written' +[ -d "$flash/styles" ] && [ -d "$flash/scripts" ] || fail 'fresh: styles and scripts folders not made' + +# An upgrade over the last stable keeps what the user saved, byte for byte, and drops the previous package +reset; install $old stable; installed $old stable; saved_config; before=$(config_sums) +install $new upgrade; installed $new upgrade +[ "$(config_sums)" = "$before" ] || fail 'upgrade: saved folders, settings, theme or Unraid orders changed' +[ ! -e "$flash/$name-$old-x86_64-1.txz" ] || fail 'upgrade: previous archive left on the flash' +[ ! -e "/var/log/packages/$name-$old-x86_64-1" ] || fail 'upgrade: previous package record left' + +# A reboot after the upgrade brings back the new version from its archive +boot boot; installed $new boot +[ "$(config_sums)" = "$before" ] || fail 'boot: saved files changed' + +# Removal leaves nothing, a file the package did not ship included, and takes only FolderView3's entries out of Unraid's +# orders, line endings kept +printf 'x' > "$live/not-from-the-package" +plugin remove $name.plg > "$work/remove.log" 2>&1 || { cat "$work/remove.log"; fail 'remove: refused'; } +[ ! -e "$live" ] && [ ! -e "$flash" ] && [ ! -e /var/log/plugins/$name.plg ] && [ ! -e "$manifest" ] || fail 'remove: files left behind' +ls /var/log/packages/$name-* >/dev/null 2>&1 && fail 'remove: package record left' +cmp -s /boot/config/plugins/dockerMan/userprefs.cfg <(printf '0="alpha"\n1="folder-sync"\n2="folder-nightlybackups"\n3="beta"\n') \ + || fail "remove: Docker order not as expected: $(od -c /boot/config/plugins/dockerMan/userprefs.cfg | head -5)" +cmp -s /boot/config/plugins/dynamix.vm.manager/userprefs.cfg <(printf '0="Debian 12"\r\n1="Windows"') \ + || fail "remove: VM order not as expected: $(od -c /boot/config/plugins/dynamix.vm.manager/userprefs.cfg | head -5)" + +# A download that fails (no network here) leaves the running install, what the user saved, and a next boot that brings it back +reset; install $old stable2; saved_config; before=$(config_sums) +plugin install "$work/$new/$name.plg" > "$work/download.log" 2>&1 && fail 'download: install reported success without its package' +grep -q 'download failure' "$work/download.log" || { cat "$work/download.log"; fail 'download: refused for a reason other than the download'; } +installed $old download +[ "$(config_sums)" = "$before" ] || fail 'download: saved files changed' +[ -f "$flash/$name-$old-x86_64-1.txz" ] || fail 'download: deleted the archive the next boot installs from' +boot download-boot; installed $old download-boot +[ "$(config_sums)" = "$before" ] || fail 'download-boot: saved files changed' + +# A failed upgradepkg does the same +reset; install $old stable3; saved_config; before=$(config_sums) +cp "$work/$new/dist/$name-$new-x86_64-1.txz" "$flash/" +mv /sbin/upgradepkg "$work/upgradepkg"; printf '#!/bin/bash\nexit 1\n' > /sbin/upgradepkg; chmod +x /sbin/upgradepkg +plugin install "$work/$new/$name.plg" > "$work/upgradepkg.log" 2>&1; status=$? +mv "$work/upgradepkg" /sbin/upgradepkg +[ $status -ne 0 ] || fail 'upgradepkg: install reported success though upgradepkg failed' +grep -q "run failed: 'upgradepkg" "$work/upgradepkg.log" || { cat "$work/upgradepkg.log"; fail 'upgradepkg: refused before upgradepkg ran'; } +installed $old upgradepkg +[ "$(config_sums)" = "$before" ] || fail 'upgradepkg: saved files changed' +[ -f "$flash/$name-$old-x86_64-1.txz" ] || fail 'upgradepkg: deleted the archive the next boot installs from' +boot upgradepkg-boot; installed $old upgradepkg-boot +[ "$(config_sums)" = "$before" ] || fail 'upgradepkg-boot: saved files changed' + +# The installer itself refuses an Unraid below the .plg's min, and says why +reset; set_version 6.12.15; cp "$work/$new/dist/$name-$new-x86_64-1.txz" "$flash/" +plugin install "$work/$new/$name.plg" > "$work/min.log" 2>&1 && fail "min: installed on 6.12.15 though min is $min" +grep -q "installed Unraid version is too low, require at least version $min" "$work/min.log" || { cat "$work/min.log"; fail 'min: no reason given'; } +[ ! -e "$live" ] || fail 'min: files installed below min' +set_version "$version" + +echo "Unraid $version: install, upgrade, boot, remove, failed download, failed upgradepkg and min passed" diff --git a/.github/unraid-compat/matrix_check.py b/.github/unraid-compat/matrix_check.py new file mode 100644 index 00000000..23c1b1e0 --- /dev/null +++ b/.github/unraid-compat/matrix_check.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""ci.yml's Unraid matrix has one release row per minor from the .plg's min to the newest, and one for any newer release.""" +import pathlib +import re +import sys + +# Only a row that YAML reads: first on its line, so a commented-out row is not counted as tested +ROW = re.compile(r"^[ \t]*- \{unraid: '([^']+)', php: '([^']+)'(?:, row: '([^']+)')?\}", re.M) +RELEASE = re.compile(r'(\d+)\.(\d+)\.\d+') + + +def problems(ci, plg): + floor = re.search(r']*\bmin="(\d+)\.(\d+)\.\d+"', plg) + if not floor: + return ['folder.view3.plg names no min="x.y.z"'] + floor = (int(floor[1]), int(floor[2])) + rows = ROW.findall(ci) + releases = [version for version, _, row in rows if not row] + tags = [version for version, _, row in rows if row == 'newest tag'] + out = [] if len(tags) == 1 else [f"ci.yml needs exactly one 'newest tag' row, not {len(tags)}"] + minors = {} + for version in releases: + m = RELEASE.fullmatch(version) + if not m: + out.append(f'release row {version} is not a release version x.y.z') + else: + minors.setdefault((int(m[1]), int(m[2])), []).append(version) + if not minors: + return out + ['ci.yml has no Unraid release rows'] + for (major, minor), versions in sorted(minors.items()): + if len(versions) > 1: + out.append(f'Unraid {major}.{minor} has {len(versions)} release rows: {", ".join(versions)}') + if (major, minor) < floor: + out.append(f'release row {versions[0]} is below the .plg\'s min {floor[0]}.{floor[1]}') + # From the min's own minor: a major with no rows at all still needs its first minor + for major in range(floor[0], max(minors)[0] + 1): + first = floor[1] if major == floor[0] else 0 + present = [n for m, n in minors if m == major] + missing = [f'{major}.{n}' for n in range(first, max(present, default=first + 1)) if n not in present] + out += [f'no release row for Unraid {v}, between the .plg\'s min and the newest row' for v in missing] + # The 'newest tag' row runs only betas; a release it reaches needs a row of its own and the PHP that release ships + out += [f'Unraid {tag} is out: give it a release row with the PHP its release notes name' for tag in tags + if RELEASE.fullmatch(tag) and tag not in releases] + return out + + +if __name__ == '__main__': + repo = pathlib.Path(__file__).resolve().parents[2] + found = problems((repo / '.github/workflows/ci.yml').read_text(encoding='utf-8'), (repo / 'folder.view3.plg').read_text(encoding='utf-8')) + print('\n'.join(found) or 'The Unraid matrix covers every minor from min to the newest release') + sys.exit(1 if found else 0) diff --git a/.github/unraid-compat/matrix_check_test.py b/.github/unraid-compat/matrix_check_test.py new file mode 100644 index 00000000..5f63c9f6 --- /dev/null +++ b/.github/unraid-compat/matrix_check_test.py @@ -0,0 +1,43 @@ +import unittest + +import matrix_check + +PLG = '' + + +def ci(*rows, tag="'7.4.0', php: '8.4'"): + lines = [f" - {{unraid: '{v}', php: '8.4'}}" for v in rows] + return '\n'.join(lines + [f" - {{unraid: {tag}, row: 'newest tag'}}"]) + + +class MatrixCheckTest(unittest.TestCase): + def test_one_row_per_minor_up_to_the_newest_passes_with_a_beta_or_that_release_as_newest_tag(self): + self.assertEqual(matrix_check.problems(ci('7.0.1', '7.1.4', '7.2.7', '7.3.5', '7.4.0'), PLG), []) + self.assertEqual(matrix_check.problems(ci('7.0.1', '7.1.4', '7.2.7', '7.3.5', '7.4.0', tag="'7.5.0-beta.1', php: '8.4'"), PLG), []) + + def test_a_release_the_newest_tag_reached_needs_its_own_row(self): + self.assertEqual(matrix_check.problems(ci('7.0.1', '7.1.4', '7.2.7', '7.3.5', '7.4.0', tag="'7.5.0', php: '8.4'"), PLG), + ['Unraid 7.5.0 is out: give it a release row with the PHP its release notes name']) + + def test_a_skipped_minor_a_second_row_for_one_and_a_row_below_min_are_named(self): + self.assertEqual(matrix_check.problems(ci('6.12.15', '7.0.1', '7.0.0', '7.2.7', '7.3.5', '7.4.0'), PLG), [ + 'release row 6.12.15 is below the .plg\'s min 7.0', + 'Unraid 7.0 has 2 release rows: 7.0.1, 7.0.0', + 'no release row for Unraid 7.1, between the .plg\'s min and the newest row']) + + def test_a_commented_out_row_is_not_tested_so_not_counted(self): + text = ci('7.0.1', '7.1.4', '7.3.5', '7.4.0').replace(" - {unraid: '7.3.5'", " # - {unraid: '7.2.9', php: '8.4'}\n - {unraid: '7.3.5'") + self.assertEqual(matrix_check.problems(text, PLG), ["no release row for Unraid 7.2, between the .plg's min and the newest row"]) + + def test_the_mins_own_major_needs_rows_when_only_a_later_major_has_them(self): + self.assertEqual(matrix_check.problems(ci('8.0.0', tag="'8.0.0', php: '8.4'"), PLG), + ["no release row for Unraid 7.0, between the .plg's min and the newest row"]) + + def test_a_missing_min_newest_tag_or_release_row_fails_closed(self): + self.assertEqual(matrix_check.problems(ci('7.0.1'), ''), ['folder.view3.plg names no min="x.y.z"']) + self.assertEqual(matrix_check.problems("- {unraid: '7.0.1', php: '8.3'}", PLG), ["ci.yml needs exactly one 'newest tag' row, not 0"]) + self.assertEqual(matrix_check.problems(ci(), PLG), ['ci.yml has no Unraid release rows']) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d682578..1199e8eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,33 +31,24 @@ jobs: - name: Install tools run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libxml2-utils - - name: PHP syntax check - run: | - status=0 - while IFS= read -r f; do - php -l "$f" || status=1 - done < <(find src -name '*.php') - exit $status - - - name: Update status test - run: php .github/scripts/update_status_test.php - - - name: FolderView Plus import tests - run: php .github/scripts/foreign_import_test.php + # php -l on every .php and .page file, then every .github/scripts/*_test.php; a deprecation fails too + - name: PHP syntax check and PHP tests + run: bash .github/scripts/php_tests.sh - - name: Autostart order tests - run: php .github/scripts/page_order_test.php - - - name: POST endpoint answer tests - run: php .github/scripts/post_answers_test.php + - name: Every !important has its reason + working-directory: .github/scripts + run: | + python3 -B -m unittest important_check_test + python3 -B important_check.py - - name: Folder file lock tests - run: php .github/scripts/folder_lock_test.php + - name: The Unraid matrix has a row for every minor up to the newest release + working-directory: .github/unraid-compat + run: | + python3 -B -m unittest matrix_check_test + python3 -B matrix_check.py - name: Variables tab tests - run: | - php .github/scripts/variables_tab_test.php - node .github/scripts/variables_tab_test.js + run: node .github/scripts/variables_tab_test.js - name: Autostart tab tests run: node .github/scripts/autostart_api_test.js @@ -146,29 +137,138 @@ jobs: python -m unittest discover -s .github/unraid-watch -p 'test_*.py' python .github/unraid-watch/watch.py check - # A dispatched run's checks never reach the PR, so vendor-sync's run reports Lint + build there as a commit status. + php: + name: PHP ${{ matrix.php }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Unraid ships 8.3 up to 7.2.7 and in 7.3.0, 8.4 from 7.2.8, 7.3.1 and 7.4; 8.5 is ahead of every release + php: ['8.3', '8.4', '8.5'] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: PHP syntax check and PHP tests + env: + PHP_VERSION: ${{ matrix.php }} + run: docker run --rm -v "$PWD:/repo" -w /repo "php:${PHP_VERSION}-cli" bash .github/scripts/php_tests.sh + + unraid: + name: Unraid ${{ matrix.unraid }}${{ matrix.row && format(' ({0})', matrix.row) || '' }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # One release row per minor from the .plg's min, on the PHP its release notes name; Renovate moves each to its newest + # patch, and the 'newest tag' row to any newer tag. matrix_check.py fails when a release has no row of its own. + include: + - {unraid: '7.0.1', php: '8.3'} + - {unraid: '7.1.4', php: '8.3'} + - {unraid: '7.2.9', php: '8.4'} + - {unraid: '7.3.5', php: '8.4'} + - {unraid: '7.4.0', php: '8.4'} + # Runs only while it names a beta or release candidate; one on a newer PHP needs its php raised by hand + - {unraid: '7.4.0', php: '8.4', row: 'newest tag'} + env: + RUNS: ${{ !matrix.row || contains(matrix.unraid, '-') }} + steps: + - name: Nothing newer than the release rows + if: env.RUNS != 'true' + env: + UNRAID_VERSION: ${{ matrix.unraid }} + run: echo "::notice::Unraid $UNRAID_VERSION is a release, so its own release row tests it" + + - name: Checkout + if: env.RUNS == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Checkout the last stable release + if: env.RUNS == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + ref: main + path: .stable + + - name: Checkout the Unraid webGUI at this version + if: env.RUNS == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + repository: unraid/webgui + ref: ${{ matrix.unraid }} + path: .webgui + sparse-checkout: | + emhttp/plugins/dynamix + emhttp/plugins/dynamix.plugin.manager + + - name: Install, upgrade, remove and failures through Unraid's own plugin installer + if: env.RUNS == 'true' + env: + PHP_VERSION: ${{ matrix.php }} + UNRAID_VERSION: ${{ matrix.unraid }} + run: | + docker build -q -t fv3-unraid --build-arg PHP="$PHP_VERSION" .github/unraid-compat + docker run --rm --network none -v "$PWD:/repo" -w /repo fv3-unraid bash .github/unraid-compat/install_test.sh .webgui "$UNRAID_VERSION" .stable + + # One check per matrix for beta's branch protection: the version rows rename themselves as Renovate moves them + php-compat: + name: PHP compatibility + needs: php + if: always() + runs-on: ubuntu-latest + steps: + - name: Every PHP version passed + env: + RESULT: ${{ needs.php.result }} + run: '[ "$RESULT" = success ]' + + unraid-compat: + name: Unraid compatibility + needs: unraid + if: always() + runs-on: ubuntu-latest + steps: + - name: Every Unraid version passed + env: + RESULT: ${{ needs.unraid.result }} + run: '[ "$RESULT" = success ]' + + # A dispatched run's checks never reach the PR, so vendor-sync's run reports each gate there as a commit status. report: - needs: validate + needs: [validate, php-compat, unraid-compat] if: always() && github.event_name == 'workflow_dispatch' && inputs.base != '' runs-on: ubuntu-latest permissions: statuses: write steps: - - name: Post the result on the PR's commit + - name: Post the results on the PR's commit env: - RESULT: ${{ needs.validate.result }} + LINT: ${{ needs.validate.result }} + PHP: ${{ needs.php-compat.result }} + UNRAID: ${{ needs.unraid-compat.result }} GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} SHA: ${{ github.sha }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail - state=failure - [ "$RESULT" = success ] && state=success - gh api "repos/$REPO/statuses/$SHA" -f state="$state" -f context='Lint + build' -f target_url="$RUN_URL" -f description="CI dispatched for this PR: $RESULT" >/dev/null + post() { + local state=failure + [ "$2" = success ] && state=success + gh api "repos/$REPO/statuses/$SHA" -f state="$state" -f context="$1" -f target_url="$RUN_URL" -f description="CI dispatched for this PR: $2" >/dev/null + } + post 'Lint + build' "$LINT" + post 'PHP compatibility' "$PHP" + post 'Unraid compatibility' "$UNRAID" notify-failure: - needs: validate + needs: [validate, php, unraid] if: failure() && github.event_name != 'pull_request' && !inputs.base permissions: {} uses: chodeus/chodeus-ops/.github/workflows/notify-discord.yml@65484add1113c1bbc00827af7237ebe7a1837bbb # main diff --git a/folder.view3.plg b/folder.view3.plg index 572bfdcf..b907aafb 100644 --- a/folder.view3.plg +++ b/folder.view3.plg @@ -857,12 +857,6 @@ After updating, hard-refresh your browser (Ctrl/Cmd+Shift+R) and clear its cache - Initial Release - - - rm -f $(ls /boot/config/plugins/&name;/&name;*.txz 2>/dev/null | grep -v '&pkgname;') - - - https://github.com/&github;/releases/download/v&version;/&pkgname;.txz @@ -872,6 +866,8 @@ After updating, hard-refresh your browser (Ctrl/Cmd+Shift+R) and clear its cache + # Only once the new package is in: a failed download or upgradepkg keeps the archive the next boot installs from + rm -f $(ls /boot/config/plugins/&name;/&name;*.txz 2>/dev/null | grep -v '&pkgname;') for p in /var/log/packages/&name;-*; do [ -e "$p" ] || continue case "$(basename "$p")" in &pkgname;) continue;; esac diff --git a/renovate.json b/renovate.json index 42aeef79..ab655f85 100644 --- a/renovate.json +++ b/renovate.json @@ -4,6 +4,26 @@ "forkProcessing": "enabled", "baseBranchPatterns": ["beta"], "customManagers": [ + { + "customType": "regex", + "description": "Each release row of the CI matrix's Unraid list, moved to the newest patch of its minor.", + "managerFilePatterns": ["/^\\.github/workflows/ci\\.yml$/"], + "matchStrings": ["- \\{unraid: '(?[^']+)', php: '[^']+'\\}"], + "datasourceTemplate": "github-tags", + "depNameTemplate": "unraid/webgui", + "depTypeTemplate": "unraid-release", + "versioningTemplate": "semver" + }, + { + "customType": "regex", + "description": "The matrix's 'newest tag' row: the newest Unraid tag of any kind, betas and release candidates included.", + "managerFilePatterns": ["/^\\.github/workflows/ci\\.yml$/"], + "matchStrings": ["- \\{unraid: '(?[^']+)', php: '[^']+', row: 'newest tag'\\}"], + "datasourceTemplate": "github-tags", + "depNameTemplate": "unraid/webgui", + "depTypeTemplate": "unraid-newest-tag", + "versioningTemplate": "semver" + }, { "customType": "regex", "description": "actionlint's release tag and tarball SHA-256 in validate-workflows.yml; Renovate takes the same asset's hash from the new release.", @@ -37,6 +57,29 @@ } ], "packageRules": [ + { + "description": "Release rows take Unraid releases only (webgui also carries 2014.x date tags, which sort above every 7.x), within their minor.", + "matchDepTypes": ["unraid-release"], + "allowedVersions": "/^\\d{1,2}\\.\\d+\\.\\d+$/" + }, + { + "description": "A new minor gets a row of its own by hand, so a release row never leaves its minor.", + "matchDepTypes": ["unraid-release"], + "matchUpdateTypes": ["major", "minor"], + "enabled": false + }, + { + "description": "Unraid releases and their betas and release candidates, still without the date tags.", + "matchDepTypes": ["unraid-newest-tag"], + "allowedVersions": "/^\\d{1,2}\\.\\d+\\.\\d+(-[0-9A-Za-z.]+)?$/", + "ignoreUnstable": false + }, + { + "description": "One Unraid PR, merged by hand: a patch can change the PHP a release ships (7.2.8 did), so each row's php is checked against the release notes.", + "matchDepNames": ["unraid/webgui"], + "groupName": "Unraid versions", + "automerge": false + }, { "description": "Bundled libraries ship to users, so their updates are fix(deps).", "matchDepTypes": ["bundled"], diff --git a/src/folder.view3/usr/local/emhttp/plugins/folder.view3/scripts/shared.js b/src/folder.view3/usr/local/emhttp/plugins/folder.view3/scripts/shared.js index 19411a79..241972b6 100644 --- a/src/folder.view3/usr/local/emhttp/plugins/folder.view3/scripts/shared.js +++ b/src/folder.view3/usr/local/emhttp/plugins/folder.view3/scripts/shared.js @@ -1912,6 +1912,7 @@ window.fv3InstallDockerTableWidthFix = () => { if (!storage || !storage.children.length) return; const children = Array.from(storage.children); const origStyles = children.map(c => c.getAttribute('style') || ''); + // !important: an inline !important outranks every stylesheet, so no theme or custom CSS shows a row while it is measured children.forEach(c => c.style.cssText += ';visibility:hidden !important;'); children.forEach(c => folder.parentNode.insertBefore(c, folder.nextSibling)); void tbl.offsetHeight;