Upgrade in place, depend on redis, and summarise each scan

.github/workflows/ci.yml

@@ -2,7 +2,10 @@ name: tests
 
 on:
   push:
-  pull_request:
+
+concurrency:
+  group: tests-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   python:

.github/workflows/openbsd.yml

@@ -5,14 +5,6 @@ name: openbsd
 # The daemon's ioctl, unveil and pledge code compiles on no other platform.
 
 on:
-  pull_request:
-    paths:
-      - 'server-rust/**'
-      - 'client-unbound/**'
-      - 'protocol/**'
-      - 'install-*.sh'
-      - 'tests/openbsd/**'
-      - '.github/workflows/openbsd.yml'
   push:
     paths:
       - 'server-rust/**'
@@ -35,6 +27,10 @@ on:
         description: 'Unbound to build: latest, master, or a release-* tag'
         default: 'latest'
 
+concurrency:
+  group: openbsd-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   end-to-end:
     name: OpenBSD ${{ matrix.release }}

README.md

@@ -280,9 +280,12 @@ Edit _unbound user's crontab to; import domain lists / execute 'unbound-control
 NB; If you want to keep the `unbound-control` cron commands under the _adblock user, you will need to add _adblock to _unbound group.
 
 NB; `client-unbound/tools/update_dns_blocklist.sh` is an alternative to
-unbound-adblock that ships with PFUI. It still fetches StevenBlack's hosts-format
-lists, so it converts with a hosts parser; pointing it at HaGeZi means reading the
-domain-only format instead.
+unbound-adblock that ships with PFUI. It fetches the same two HaGeZi lists,
+`pro` and `tif`, converts them to Unbound's local-zone format and publishes one
+file for `include:`. It refuses to publish a list that is short or has shrunk
+sharply, so a failed download cannot quietly turn the filter off. An existing
+copy is never replaced by the installer, since the sources and thresholds are
+meant to be edited.
 
 References;\
 HaGeZi's lists and what each tier blocks: https://github.com/hagezi/dns-blocklists \

client-unbound/tests/test_blocklist_update.py

@@ -37,8 +37,11 @@
   esac
 fi
 lines=${CURL_LINES:-1200}
-awk -v n="$lines" 'BEGIN { for (i = 0; i < n; i++) print "0.0.0.0 bad" i ".example.com" }' \
-  > "$dest"
+{
+  echo "# Title: stub list"
+  echo "!"
+  awk -v n="$lines" 'BEGIN { for (i = 0; i < n; i++) print "bad" i ".example.com" }'
+} > "$dest"
 """
 
 RECORDER_STUB = r"""#!/usr/bin/env bash
@@ -113,16 +116,16 @@ def test_norestart_publishes_without_restarting(harness):
 
 
 def test_a_failed_source_falls_back_to_the_mirror(harness):
-    """The StevenBlack lists have a second URL; using it is not a failure."""
+    """Each list has a second URL; using it is not a failure."""
     result = harness.run(CURL_FAIL="raw.githubusercontent.com")
     assert result.returncode == 0, result.stderr.decode()
-    assert "sbc.io" in harness.calls()
+    assert "jsdelivr" in harness.calls()
     assert harness.lines()
 
 
 def test_a_source_that_cannot_be_downloaded_at_all_aborts(harness):
     """No previous copy to fall back on, so there is nothing to publish."""
-    result = harness.run(CURL_FAIL="hosts")
+    result = harness.run(CURL_FAIL="onlydomains")
     assert result.returncode != 0
     assert harness.lines() is None, "published a list built from a failed download"
     assert "rcctl" not in harness.calls(), "restarted the resolver anyway"
@@ -134,7 +137,7 @@ def test_a_source_that_fails_today_is_reused_from_yesterday(harness):
     assert harness.run("norestart").returncode == 0
     before = harness.lines()
 
-    result = harness.run("norestart", CURL_FAIL="pgl.yoyo.org")
+    result = harness.run("norestart", CURL_FAIL="tif-onlydomains")
     assert result.returncode == 0, result.stderr.decode()
     assert harness.lines() == before, "reusing a cached source lost domains"
 
@@ -148,7 +151,7 @@ def test_a_previously_published_list_survives_a_failed_run(harness):
         if stale.name != "dns_blocklist":
             stale.unlink()
 
-    result = harness.run(CURL_FAIL="hosts")
+    result = harness.run(CURL_FAIL="onlydomains")
     assert result.returncode != 0
     assert harness.lines() == before, "a failed run replaced a working blocklist"
     assert "rcctl" not in harness.calls(), "restarted the resolver anyway"
@@ -194,3 +197,23 @@ def test_no_staging_files_are_left_behind(harness):
     harness.run("norestart", CURL_LINES=3)  # Refused
     leftovers = [p.name for p in harness.etc.iterdir() if ".new." in p.name]
     assert leftovers == [], f"left {leftovers} in place"
+
+
+def test_only_plausible_domains_reach_unbound(harness):
+    """HaGeZi's files carry a header, and the wildcard variants carry '*.'
+    prefixes. A line Unbound cannot parse as a local-zone takes the whole
+    include file with it, so anything that is not a domain is dropped."""
+    extra = "\n".join(
+        [
+            'echo "*.wildcard.example"',
+            'echo "0.0.0.0 hosts-format.example"',
+            'echo "not a domain at all"',
+        ]
+    )
+    stub = harness.etc.parent / "bin" / "curl"
+    stub.write_text(stub.read_text().replace("} > \"$dest\"", extra + "\n} > \"$dest\""))
+    assert harness.run("norestart").returncode == 0
+    published = "\n".join(harness.lines())
+    assert "bad1.example.com" in published
+    for rejected in ("wildcard.example", "0.0.0.0 hosts-format", "not a domain"):
+        assert rejected not in published, f"{rejected!r} reached the include file"

client-unbound/tools/update_dns_blocklist.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
-# Example DNS BlockList script to download common bad domains from some example well known sources
+# Example DNS BlockList script: HaGeZi's pro and tif domain lists, converted to
+# Unbound's local-zone format and published as one file for 'include:'.
+# https://github.com/hagezi/dns-blocklists
 
 set -u
 
@@ -52,52 +54,46 @@ require() {
   [ -s "$1" ] || die "$2 is unavailable and no previous copy exists ($1)"
 }
 
-# hosts_to_unbound <raw> <unbound-format>
-hosts_to_unbound() {
-  grep '^0\.0\.0\.0' "$1" \
-    | awk '{print "local-zone: \""$2"\" redirect\nlocal-data: \""$2" A 0.0.0.0\""}' \
+# domains_to_unbound <domain list> <unbound-format>
+# One domain per line, with comments and a header. A line Unbound cannot parse as
+# a local-zone takes the whole include file with it, so only domains are written.
+domains_to_unbound() {
+  grep -E '^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$' "$1" \
+    | awk '{print "local-zone: \""$1"\" redirect\nlocal-data: \""$1" A 0.0.0.0\""}' \
     > "$2"
 }
 
 lines() { wc -l < "$1" | tr -d '[:space:]'; }
 
 log info "Updating DNS Domain Filter Lists"
 
-# https://github.com/StevenBlack/hosts
-SB_BASE=https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates
-# Fallback is plain HTTP and unauthenticated: content is not verified
-SB_MIRROR=http://sbc.io/hosts/alternates
-
-log info "Downloading StevenBlack Bad Domains - Unified hosts (adware + malware) + fakenews + gambling"
-fetch "${ETC}/stephenblack_adware_malware_fakenews_gambling-raw" \
-  "${SB_BASE}/fakenews-gambling/hosts" \
-  "${SB_MIRROR}/fakenews-gambling/hosts"
-require "${ETC}/stephenblack_adware_malware_fakenews_gambling-raw" \
-  "StevenBlack fakenews-gambling list"
-
-log info "Downloading StevenBlack Bad Domains - Unified hosts (adware + malware) + fakenews + gambling + social"
-fetch "${ETC}/stephenblack_adware_malware_fakenews_gambling_social-raw" \
-  "${SB_BASE}/fakenews-gambling-social/hosts" \
-  "${SB_MIRROR}/fakenews-gambling-social/hosts"
-require "${ETC}/stephenblack_adware_malware_fakenews_gambling_social-raw" \
-  "StevenBlack fakenews-gambling-social list"
-
-log info "Converting StevenBlack Bad Domains from RAW format to Unbound config format"
-hosts_to_unbound "${ETC}/stephenblack_adware_malware_fakenews_gambling-raw" \
-  "${ETC}/stephenblack_adware_malware_fakenews_gambling-unbound"
-hosts_to_unbound "${ETC}/stephenblack_adware_malware_fakenews_gambling_social-raw" \
-  "${ETC}/stephenblack_adware_malware_fakenews_gambling_social-unbound"
-
-echo
-log info "Downloading YoYo AdServers Bad Domains"
-fetch "${ETC}/yoyo_adservers-unbound" \
-  "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=unbound&showintro=0&mimetype=plaintext"
-require "${ETC}/yoyo_adservers-unbound" "YoYo AdServers list"
+# https://github.com/hagezi/dns-blocklists
+# The -onlydomains variants only: wildcard/*.txt carries '*.' prefixes and
+# adblock/ carries ABP filter syntax, neither of which Unbound loads.
+HG_BASE=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard
+# Same content, used only if the first fails
+HG_MIRROR=https://cdn.jsdelivr.net/gh/hagezi/dns-blocklists@latest/wildcard
+
+# pro: ads, tracking and telemetry. tif: malware, phishing and scams.
+for list in pro tif; do
+  log info "Downloading HaGeZi ${list} domains"
+  fetch "${ETC}/hagezi_${list}-raw" \
+    "${HG_BASE}/${list}-onlydomains.txt" \
+    "${HG_MIRROR}/${list}-onlydomains.txt"
+  require "${ETC}/hagezi_${list}-raw" "HaGeZi ${list} list"
+done
+
+log info "Converting HaGeZi domains to Unbound config format"
+for list in pro tif; do
+  domains_to_unbound "${ETC}/hagezi_${list}-raw" "${ETC}/hagezi_${list}-unbound"
+  [ -s "${ETC}/hagezi_${list}-unbound" ] \
+    || die "the HaGeZi ${list} list produced no usable domains"
+done
 
 echo
 log info "Merging all Bad Domains"
-cat "${ETC}/stephenblack_adware_malware_fakenews_gambling_social-unbound" \
-    "${ETC}/yoyo_adservers-unbound" > "${ETC}/dns_blocklist_all" \
+cat "${ETC}/hagezi_pro-unbound" "${ETC}/hagezi_tif-unbound" \
+    > "${ETC}/dns_blocklist_all" \
   || die "cannot write ${ETC}/dns_blocklist_all"
 
 echo

install-client-unbound.sh

@@ -381,8 +381,16 @@ if [[ "$OS" = "OpenBSD" ]]; then
   [ -f "${TARGET}/update_root_hints.sh" ] && mv "${TARGET}/update_root_hints.sh" "${TARGET}/update_root_hints.sh.${HOUR}"
   # root-owned: these run from cron with the privilege to write /var/unbound and restart the service
   install -m 755 -o root -g wheel "${DIR}"/client-unbound/tools/update_root_hints.sh ${TARGET}/update_root_hints.sh
-  [ -f "${TARGET}/update_dns_blocklist.sh" ] && mv "${TARGET}/update_dns_blocklist.sh" "${TARGET}/update_dns_blocklist.sh.${HOUR}"
-  install -m 755 -o root -g wheel "${DIR}"/client-unbound/tools/update_dns_blocklist.sh ${TARGET}/update_dns_blocklist.sh
+  # The sources, allowlist hook and thresholds are meant to be edited, so an
+  # existing copy is kept
+  if [ -f "${TARGET}/update_dns_blocklist.sh" ]; then
+    echo "PFUIDNS: Keeping the existing ${TARGET}/update_dns_blocklist.sh"
+    echo "PFUIDNS: (the shipped version is ${DIR}/client-unbound/tools/update_dns_blocklist.sh)"
+  else
+    install -m 755 -o root -g wheel "${DIR}"/client-unbound/tools/update_dns_blocklist.sh \
+      ${TARGET}/update_dns_blocklist.sh || die "cannot install update_dns_blocklist.sh"
+    echo "PFUIDNS: Installed ${TARGET}/update_dns_blocklist.sh (HaGeZi pro + tif)"
+  fi
   echo "New scripts: ${TARGET}/update_root_hints.sh, ${TARGET}/update_dns_blocklist.sh"
 
   # The resolver's own config carries an operator's whole ruleset, so an

install-server-python.sh

@@ -26,6 +26,32 @@ add_pkg() {
   done
 }
 
+# install(1) renames into place, so a binary replaced under a running daemon
+# leaves that daemon on the old inode.
+SERVICE_WAS_RUNNING=no
+
+stop_for_upgrade() {
+  rcctl check pfui_firewall >/dev/null 2>&1 || return 0
+  SERVICE_WAS_RUNNING=yes
+  echo "PFUIFW: Stopping pfui_firewall to upgrade it"
+  rcctl stop pfui_firewall >/dev/null 2>&1 \
+    || die "cannot stop pfui_firewall; stop it by hand and re-run"
+}
+
+# Only what was running is started. A failure is loud: the firewall was
+# whitelisting a moment ago.
+start_after_upgrade() {
+  [ "${SERVICE_WAS_RUNNING}" = yes ] || return 0
+  echo "PFUIFW: Starting pfui_firewall"
+  if rcctl start pfui_firewall; then
+    return 0
+  fi
+  err=1
+  echo "PFUIFW: FAILED to start pfui_firewall, which was running before this" >&2
+  echo "        upgrade, so this firewall is not whitelisting now. Its reason:" >&2
+  echo "        tail /var/log/daemon | grep pfui_firewall" >&2
+}
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 HOUR=$(date +%d-%b-%H_%M)
 
@@ -118,6 +144,8 @@ if [[ "$OS" = "OpenBSD" ]]; then
   python3 -c "import lz4.frame, yaml, redis, service" \
     || die "dependencies are not importable by $(command -v python3)"
 
+  stop_for_upgrade
+
   echo "PFUIFW: Installing PFUI Firewall Service (will backup any existing pfui_firewall configuration)"
   install -m 755 -o root -g wheel "${DIR}"/server-python/pfui_firewall.py /usr/local/sbin/pfui_firewall \
     || die "cannot install the daemon"
@@ -186,14 +214,20 @@ if [[ "$OS" = "OpenBSD" ]]; then
   echo "PFUIFW: NOTE point pf.conf 'persist file' paths at ${PERSIST_DIR}/ipv{4,6}_domains (see /etc/pf-pfui-example.conf)"
 fi
 
+start_after_upgrade
+
 if [[ $err != 0 ]]; then
   echo "PFUIFW: All Completed, but with some errors. Please investigate."
   exit 1
 else
   echo "PFUIFW: All Completed successfully."
 fi
-echo "PFUIFW: Enable service 'rcctl enable pfui_firewall'"
-echo "PFUIFW: Start service 'rcctl start pfui_firewall'"
+if [ "${SERVICE_WAS_RUNNING}" = yes ]; then
+  echo "PFUIFW: pfui_firewall is running the version just installed"
+else
+  echo "PFUIFW: Enable service 'rcctl enable pfui_firewall'"
+  echo "PFUIFW: Start service 'rcctl start pfui_firewall'"
+fi
 echo
 echo "PFUIFW: If PFUI_Unbound runs on THIS host, uncomment SOCKET_UNIX in"
 echo "        /etc/pfui_firewall.yml and add '- SOCKET: /var/run/pfui/pfui_firewall.sock'"

install-server-rust.sh

@@ -29,6 +29,32 @@ add_pkg() {
   done
 }
 
+# install(1) renames into place, so a binary replaced under a running daemon
+# leaves that daemon on the old inode.
+SERVICE_WAS_RUNNING=no
+
+stop_for_upgrade() {
+  rcctl check pfui_firewall >/dev/null 2>&1 || return 0
+  SERVICE_WAS_RUNNING=yes
+  echo "PFUIFW: Stopping pfui_firewall to upgrade it"
+  rcctl stop pfui_firewall >/dev/null 2>&1 \
+    || die "cannot stop pfui_firewall; stop it by hand and re-run"
+}
+
+# Only what was running is started. A failure is loud: the firewall was
+# whitelisting a moment ago.
+start_after_upgrade() {
+  [ "${SERVICE_WAS_RUNNING}" = yes ] || return 0
+  echo "PFUIFW: Starting pfui_firewall"
+  if rcctl start pfui_firewall; then
+    return 0
+  fi
+  err=1
+  echo "PFUIFW: FAILED to start pfui_firewall, which was running before this" >&2
+  echo "        upgrade, so this firewall is not whitelisting now. Its reason:" >&2
+  echo "        tail /var/log/daemon | grep pfui_firewall" >&2
+}
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 HOUR=$(date +%d-%b-%H_%M)
 
@@ -131,6 +157,8 @@ fi
 # re-applies this on every start, because MAKEDEV resets it on release upgrades
 chgrp _pfui_firewall /dev/pf && chmod 660 /dev/pf || die "cannot set /dev/pf ownership"
 
+stop_for_upgrade
+
 echo "PFUIFW: Installing PFUI Firewall daemon"
 # The binary is the whole install: no module tree, no wire module. Overwrites
 # the Python daemon at the same path when present, deliberately, for drop-in.
@@ -199,14 +227,20 @@ if grep -q 'persist file "/var/spool/pfui' /etc/pf.conf 2>/dev/null; then
   echo "        Until then a PF reload restores whatever the old files hold."
 fi
 
+start_after_upgrade
+
 if [[ $err != 0 ]]; then
   echo "PFUIFW: All Completed, but with some errors. Please investigate."
   exit 1
 else
   echo "PFUIFW: All Completed successfully."
 fi
-echo "PFUIFW: Enable service 'rcctl enable pfui_firewall'"
-echo "PFUIFW: Start service 'rcctl start pfui_firewall'"
+if [ "${SERVICE_WAS_RUNNING}" = yes ]; then
+  echo "PFUIFW: pfui_firewall is running the version just installed"
+else
+  echo "PFUIFW: Enable service 'rcctl enable pfui_firewall'"
+  echo "PFUIFW: Start service 'rcctl start pfui_firewall'"
+fi
 echo
 echo "PFUIFW: If PFUI_Unbound runs on THIS host, uncomment SOCKET_UNIX in"
 echo "        /etc/pfui_firewall.yml and add '- SOCKET: /var/run/pfui/pfui_firewall.sock'"

server-python/rc.d/pfui_firewall

@@ -26,6 +26,21 @@ rc_pre() {
 		install -d -o _pfui_firewall -g _pfui_firewall -m 755 /var/run/pfui
 	fi
 	chgrp _pfui_firewall /dev/pf && chmod 660 /dev/pf
+
+	# The daemon whitelists without Redis, but nothing expires until it appears.
+	# Never fatal: a non-zero rc_pre would abort a start the daemon can manage.
+	# Only a local REDIS_HOST is ours to start.
+	redis_host=$(sed -n 's/^[[:space:]]*REDIS_HOST:[[:space:]]*\([^[:space:]#]*\).*/\1/p' \
+	    /etc/pfui_firewall.yml 2>/dev/null | tr -d "\"'" | tail -1)
+	case "${redis_host:-127.0.0.1}" in
+	127.*|::1|localhost)
+		if ! rcctl check redis >/dev/null 2>&1; then
+			echo "pfui_firewall: starting redis first"
+			rcctl start redis >/dev/null 2>&1 || \
+			    echo "pfui_firewall: redis did not start; nothing will expire until it does"
+		fi
+		;;
+	esac
 }
 
 # The daemon takes its verb positionally, so ${daemon} ${daemon_flags} cannot be