From 7b920eed884da7442fc01ad7b1c21eb0b399c116 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 10:23:18 +0200 Subject: [PATCH 1/5] Initialize deployed databases from the installed release --- deploy/README.md | 19 ++++---- .../ansible/roles/dispatcher/tasks/main.yml | 17 +++---- deploy/ansible/roles/executor/tasks/main.yml | 19 ++++---- deploy/debuglet-deploy | 2 +- deploy/test/ansible-render.sh | 47 ++++++++++++++----- docs/operations/configuration.md | 2 +- 6 files changed, 64 insertions(+), 42 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 59ce51bd..d993242c 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -224,11 +224,11 @@ lets the candidate's own installer place the payload under its own activation links, so updating dev cannot switch prod's executable. Redeploying the same release installs nothing again. -`make deploy-seed-db` writes the schema-only SQLite databases the roles -install alongside them. It runs the already-built payload's daemon commands -with `-init-database PATH`, after checking its installed manifest matches the -release in `deploy/dist`. Run `make deploy-build` first; an image from another -release is refused. Neither `deploy/dist/` nor `deploy/certs/` is committed. +For a new host, each role initializes its database with the installed release's +`-init-database PATH` command as the service account. No seed files need to be +added to a signed release bundle. Neither `deploy/dist/` nor `deploy/certs/` +is committed. `make deploy-seed-db` remains available for tools that need +separate schema-only files; the Ansible roles do not use them. ## The provisioner @@ -415,11 +415,10 @@ Executors on a shared host use separate `debuglet-prod`/`debuglet-dev` users, `debuglet-executor-.service` units. Package staging and deployment records are also separate. Prod and dev dispatchers remain on separate hosts. -A daemon never creates its database. When a host has none yet, the roles -install the schema-only seed that `make deploy-seed-db` wrote, which is the -step a daemon's absent-database refusal names. -The roles never overwrite a nonempty database with a seed, and they never -upgrade one. A zero-byte executor database can be replaced with the seed. If a +Normal daemon startup never creates its database. When a host has none yet, +the roles explicitly initialize it using the installed daemon's embedded +migrations. They never overwrite or upgrade a nonempty database. A zero-byte +executor database can be reinitialized. If a previous installation has nonempty state and the selected environment has no database yet, deployment stops and preserves that state and its legacy unit. Back the database up, place it in the selected state directory, deploy again diff --git a/deploy/ansible/roles/dispatcher/tasks/main.yml b/deploy/ansible/roles/dispatcher/tasks/main.yml index 78c9d3be..a036a252 100644 --- a/deploy/ansible/roles/dispatcher/tasks/main.yml +++ b/deploy/ansible/roles/dispatcher/tasks/main.yml @@ -36,14 +36,15 @@ upgrade-database.yml. quiet: true -- name: Provision dispatcher database (schema only, preserved on redeploy) - ansible.builtin.copy: - src: "{{ dist_dir }}/dispatcher-seed.db" # built by `make deploy-seed-db`, not committed - dest: "{{ state_dir }}/dispatcher/dispatcher.db" - owner: "{{ debuglet_user }}" - group: "{{ debuglet_group }}" - mode: "0600" - force: false # never overwrite an existing database: it holds recorded runs +- name: Initialize a missing dispatcher database with the installed release + become: true + become_user: "{{ debuglet_user }}" + ansible.builtin.command: + argv: + - "{{ payload_prefix }}/lib/debuglet/{{ debuglet_release.version }}/bin/debuglet-dispatcher" + - -init-database + - "{{ state_dir }}/dispatcher/dispatcher.db" + creates: "{{ state_dir }}/dispatcher/dispatcher.db" notify: Restart dispatcher # The RIS ASN database refresh. The dispatcher refuses to start with a diff --git a/deploy/ansible/roles/executor/tasks/main.yml b/deploy/ansible/roles/executor/tasks/main.yml index ecd6dd1c..48a4d342 100644 --- a/deploy/ansible/roles/executor/tasks/main.yml +++ b/deploy/ansible/roles/executor/tasks/main.yml @@ -56,7 +56,7 @@ - executor_legacy_unit.stat.exists # An interrupted initial start can leave a zero-byte database. It contains no -# state; unlike a nonempty database it must not block installation of the seed. +# state; unlike a nonempty database it must not block initialization. - name: Discard an empty executor database ansible.builtin.file: path: "{{ executor_state_dir }}/executor.db" @@ -65,14 +65,15 @@ - executor_db.stat.exists - executor_db.stat.size == 0 -- name: Provision executor database (schema only, preserved on redeploy) - ansible.builtin.copy: - src: "{{ dist_dir }}/executor-seed.db" # built by `make deploy-seed-db`, not committed - dest: "{{ executor_state_dir }}/executor.db" - owner: "{{ executor_user }}" - group: "{{ executor_group }}" - mode: "0600" - force: false # never overwrite an existing database: it holds recorded runs +- name: Initialize a missing executor database with the installed release + become: true + become_user: "{{ executor_user }}" + ansible.builtin.command: + argv: + - "{{ payload_prefix }}/lib/debuglet/{{ debuglet_release.version }}/bin/debuglet-executor" + - -init-database + - "{{ executor_state_dir }}/executor.db" + creates: "{{ executor_state_dir }}/executor.db" notify: Restart executor # The rendered configuration carries the TESLA seed, so its content stays out diff --git a/deploy/debuglet-deploy b/deploy/debuglet-deploy index 8faa8dd2..944018e5 100755 --- a/deploy/debuglet-deploy +++ b/deploy/debuglet-deploy @@ -92,7 +92,7 @@ if [ "$action" = certs ]; then exit fi -make -C "$root" deploy-build deploy-seed-db +make -C "$root" deploy-build deploy_version=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["version"])' \ "$root/deploy/dist/release.json") extra+=(-e "deploy_version=$deploy_version") diff --git a/deploy/test/ansible-render.sh b/deploy/test/ansible-render.sh index 9e88cbca..201f5ddd 100755 --- a/deploy/test/ansible-render.sh +++ b/deploy/test/ansible-render.sh @@ -12,7 +12,7 @@ # origin and a non-UUID executor identity, and accepts a complete set, # 3. the playbooks apply in the documented order — deploy-certs.yml, then # site.yml — and the roles create the directory skeleton, install the -# verified release package with its own installer, install the seed +# verified release package with its own installer, initialize missing # databases and render both daemon configurations and both systemd units, # 4. the rendered configurations name the configured listener addresses and # ports, carry explicit wallet-free payment settings, and put each @@ -106,17 +106,13 @@ certs=$work/certs # fixture provides it the same way, so the roles install into it rather than # create it. mkdir -p "$host" "$dist" "$host/etc/systemd/system" -# The real release package, plus stand-ins for the schema-only databases -# `make deploy-seed-db` writes. The package is installed by its own installer, -# which verifies it; the two database files are only copied into place. +# The real release package, without separately generated seed databases. +# Each installed daemon initializes its own schema on the fixture host. cp "$root/deploy/dist/release.json" "$root/deploy/dist/SHA256SUMS" \ "$root/deploy/dist/install.sh" "$dist/" release_version=$(tr -d ' \n' <"$dist/release.json" | grep -o '"version":"[^"]*"' | cut -d'"' -f4) release_archive=$(tr -d ' \n' <"$dist/release.json" | grep -o '"archive":"[^"]*"' | cut -d'"' -f4) cp "$root/deploy/dist/$release_archive" "$dist/" -for artifact in dispatcher-seed.db executor-seed.db; do - printf 'fixture %s\n' "$artifact" >"$dist/$artifact" -done # The preflight requires pinned host identities before anything connects. The # fixture host is reached over the local connection and never over SSH, but @@ -498,6 +494,34 @@ fi dispatcher_toml=$host/etc/debuglet/dispatcher/dispatcher.toml executor_toml=$host/etc/debuglet/executor-prod/executor.toml +databases=("$host/var/lib/debuglet/dispatcher/dispatcher.db" "$host/var/lib/debuglet/executor-prod/executor.db") +if python3 - "${databases[@]}" <<'PY' +import sqlite3, sys +for path, table in zip(sys.argv[1:], ('transactions', 'debuglets')): + with sqlite3.connect('file:' + path + '?mode=rw', uri=True) as db: + assert db.execute('PRAGMA quick_check').fetchone() == ('ok',) + assert db.execute('SELECT name FROM sqlite_master WHERE name=?', (table,)).fetchone() + db.execute('CREATE TABLE deployment_retention_fixture(value TEXT)') + db.execute("INSERT INTO deployment_retention_fixture VALUES ('preserved')") +PY +then + check 'installed daemons initialize valid databases without seed files' pass +else + check 'installed daemons initialize valid databases without seed files' fail +fi +sha256sum "${databases[@]}" >"$work/databases-before" +if run "$work/state-redeploy.log" site.yml && + sha256sum --check "$work/databases-before" >/dev/null; then + check 'redeployment preserves populated databases byte for byte' pass +else + check 'redeployment preserves populated databases byte for byte' fail + tail -20 "$work/state-redeploy.log" >&2 +fi +# The configuration/TLS checks below deliberately reach an invalid database, +# so a successful parse cannot accidentally start a long-running daemon. +for database in "${databases[@]}"; do + printf 'fixture database\n' >"$database" +done for file in "$dispatcher_toml" "$executor_toml" \ "$host/etc/systemd/system/debuglet-dispatcher.service" \ "$host/etc/systemd/system/debuglet-executor-prod.service" \ @@ -978,7 +1002,8 @@ rm -f "$host/opt/debuglet/dev/bin/debuglet-executor" if run "$work/dev-repair.log" deploy-executors.yml --limit executors \ -e @vars/dev.yml -e "executor_id=$fixture_dev_executor" && [ -x "$host/opt/debuglet/dev/bin/debuglet-executor" ] && - cmp -s "$dist/executor-seed.db" "$host/var/lib/debuglet/executor-dev/executor.db"; then + "$host/opt/debuglet/dev/bin/debuglet-executor" -check-database \ + -config "$host/etc/debuglet/executor-dev/executor.toml" >"$work/dev-db-check.log" 2>&1; then check 'dev repairs its own activation link and empty database' pass else check 'dev repairs its own activation link and empty database' fail @@ -1153,13 +1178,9 @@ refute 'no enrollment token is rendered by default' "$executor_toml" 'enrollment bind_host=$work/bind-host bind_certs=$work/bind-certs bind_dist=$work/bind-dist -mkdir -p "$bind_host/etc/systemd/system" "$bind_dist" "$work/bind-seed" -chmod 0700 "$work/bind-seed" +mkdir -p "$bind_host/etc/systemd/system" "$bind_dist" cp -R "$certs" "$bind_certs" cp "$dist"/* "$bind_dist/" -rm -f "$bind_dist/dispatcher-seed.db" -"$host/opt/debuglet/prod/bin/debuglet-dispatcher" -init-database "$work/bind-seed/dispatcher.db" \ - >"$work/bind-seed.log" 2>&1 && cp "$work/bind-seed/dispatcher.db" "$bind_dist/dispatcher-seed.db" cat >"$work/bind.yml" < Date: Thu, 15 Oct 2026 10:16:30 +0200 Subject: [PATCH 2/5] Clarify exact-version Linux candidate installation --- README-install.md | 123 +++++++++++++++++++++++++++++++++------- scripts/bootstrap.sh | 14 ++++- tools/test_bootstrap.py | 41 +++++++++++++- 3 files changed, 153 insertions(+), 25 deletions(-) diff --git a/README-install.md b/README-install.md index 3319a65a..21b30e42 100644 --- a/README-install.md +++ b/README-install.md @@ -1,22 +1,90 @@ # Install Debuglet -Debuglet releases support Linux amd64. For the full bundle, download the archive, `install.sh`, and `SHA256SUMS` from the [latest release](https://github.com/netsec-ethz/debuglet/releases/latest), then install to your user directory: +Use Linux amd64 and the exact package version recommended by your dispatcher +operator. For executor enrollment, managed services, and the local demo, choose +the **full bundle**. The CLI-only and daemon-only packages do not provide that +complete setup flow. + +The current executor setup flow requires a newer package than the published, +unsigned v0.2.0 release. Until a compatible signed release is published, your +operator must supply a private development candidate. A development candidate +is not a public release and has no GitHub release download URL. + +## Install a private development candidate + +No source checkout or compiler is required. Obtain these files, the exact +candidate version, and its full source revision through your operator's trusted +channel: + +- A trusted standalone copy of `bootstrap.sh` from the candidate source. +- `debuglet-VERSION-linux-amd64.tar.gz`, `install.sh`, and `SHA256SUMS`, kept + together in one local directory with their original filenames. + +The bootstrap verifies the archive and installer checksums **before executing +the installer**. The installer then verifies the package contents and installed +files. These checks detect changed bytes; an unsigned candidate does not +authenticate its publisher. Trust in both the bootstrap and the candidate comes +from the channel through which your operator supplies them. + +Set the exact values supplied by your operator. `VERSION_PROVIDED_BY_OPERATOR` +below is a placeholder for a version such as `v0.0.0-dev.0123456789ab`; do not +rename an archive or infer its version from an unrelated checkout. ```sh -archive=$(printf '%s\n' ./debuglet-v*-linux-amd64.tar.gz) -version=${archive#./debuglet-} -version=${version%-linux-amd64.tar.gz} -sh ./install.sh --archive "$archive" --checksums ./SHA256SUMS \ - --version "$version" --prefix "$HOME/.local" +DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR \ +DEBUGLET_RELEASE_DIR="$HOME/Downloads/debuglet-candidate" \ +DEBUGLET_ALLOW_UNSIGNED=1 \ +DEBUGLET_PREFIX="$HOME/.local" \ +sh "$HOME/Downloads/bootstrap.sh" export PATH="$HOME/.local/bin:$PATH" +dbl --output json version dbl demo ``` -The installer verifies the release files before writing them. It requires a POSIX shell, GNU tar, and coreutils. The demo needs no Go compiler, source checkout, service, account, wallet, or root access. +Use your actual local paths. Leave `DEBUGLET_COMPONENT` unset for the full +bundle. The command refuses development candidates without a local package +directory and explicit unsigned opt-in. Signature requirements or configured +release trust cannot be overridden by unsigned mode. + +Confirm that `dbl version` reports the supplied version and source revision. +The demo needs no service, account, wallet, or root access. Installing the +package does not enroll an executor or create a service account; continue with +[executor setup](docs/operations/executor-onboarding.md), or use +[managed services](docs/operations/services.md) for a system-wide installation. + +The installer requires a POSIX shell, GNU tar, and coreutils. Its versioned +files are under `PREFIX/lib/debuglet/VERSION`, and `PREFIX/bin/dbl` selects the +installed bundle. Installing another version does not restart an existing +service; follow the managed-service update procedure. + +## Install a signed release + +Use this route only after your operator identifies an exact published version +with downloadable signed assets. Obtain the standalone bootstrap and provision +the release signer's trust file independently, as described in +[Signed releases and offline verification](docs/operations/releases.md). +This needs curl, Python 3.11 or later, and OpenSSH with `ssh-keygen -Y` support +in addition to the installer prerequisites. + +```sh +DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR \ +DEBUGLET_REQUIRE_SIGNATURE=1 \ +DEBUGLET_RELEASE_TRUST=/etc/debuglet/release-allowed-signers \ +DEBUGLET_RELEASE_SIGNER=releases@debuglet.netsec.ethz.ch \ +DEBUGLET_PREFIX="$HOME/.local" \ +sh /path/to/trusted/bootstrap.sh +``` + +Leave `DEBUGLET_RELEASE_DIR` unset for public release downloads. For a signed +offline install, set it to the directory containing the selected package files, +`release.json`, and `release.json.sig`. The bootstrap verifies the signature +before executing the installer and never falls back to unsigned mode. -For signed releases, provision trust and verify the installer before execution -using [Signed releases and offline verification](docs/operations/releases.md). -That guide also describes the explicit mode for existing unsigned packages. +For the older public demo only, a trusted bootstrap can download the existing +unsigned package with `DEBUGLET_VERSION=v0.2.0 DEBUGLET_ALLOW_UNSIGNED=1 sh +/path/to/trusted/bootstrap.sh`. Leave the component, local-directory and +signature settings unset for that separate legacy invocation. v0.2.0 does not +provide `dbl executor join`. ## Supported platforms @@ -54,7 +122,13 @@ make ci-build make ci-package ``` -The full bundle is under `.cache/ci/packages/`. Use it with the same installer command above. The build also produces the component packages below. +The full bundle is under `.cache/ci/packages/`. The build prints its exact +archive name, including the generated `v0.0.0-dev.` version. Run +the development-candidate command above with that version, +`DEBUGLET_RELEASE_DIR="$PWD/.cache/ci/packages"`, and `sh scripts/bootstrap.sh`. +For another machine, copy the trusted bootstrap and the three package files; +the source checkout stays on the build machine. The build also produces the +component packages below. The package output includes `compatibility.json` and `SHA256SUMS-compatibility`. This separate record identifies the core package, @@ -73,20 +147,29 @@ Component packages are available from current source builds and are intended for | Executor operators | `executor` | `.cache/ci/packages/executor/` | `debuglet-executor` | | Dispatcher operators | `dispatcher` | `.cache/ci/packages/dispatcher/` | `debuglet-dispatcher` | -Choose a component, then run this from the source checkout after building: +Obtain the selected component's three files and the trusted bootstrap from +your operator, or use the source build directory above. Select the same exact +version as the full bundle: ```sh -component=cli # or executor or dispatcher -cd ".cache/ci/packages/$component" -archive=$(printf '%s\n' ./debuglet-"$component"-v*-linux-amd64.tar.gz) -version=${archive#./debuglet-$component-} -version=${version%-linux-amd64.tar.gz} -sh "./install-$component.sh" --archive "$archive" \ - --checksums "./SHA256SUMS-$component" --version "$version" --prefix "$HOME/.local" +DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR \ +DEBUGLET_COMPONENT=cli \ +DEBUGLET_RELEASE_DIR="$HOME/Downloads/debuglet-candidate/cli" \ +DEBUGLET_ALLOW_UNSIGNED=1 \ +DEBUGLET_PREFIX="$HOME/.local" \ +sh "$HOME/Downloads/bootstrap.sh" export PATH="$HOME/.local/bin:$PATH" ``` -Each directory contains only its archive, installer and checksums. These filenames are also unique release asset names. Role installations share a prefix safely: upgrading one changes only its own command link. The daemon packages include their SQLite support and embedded migrations; they do not require `goose` or `sqlc` at runtime. Daemons still need configuration and a prepared database; see the operator guide below. +Replace `cli` with `executor` or `dispatcher` and use its matching directory +when needed. Each directory contains its archive, `install-COMPONENT.sh`, and +`SHA256SUMS-COMPONENT`. These filenames are also unique release asset names. +For signed components, use the signed-release command with +`DEBUGLET_COMPONENT` instead of unsigned mode. Role installations share a prefix +safely: upgrading one changes only its own command link. The daemon packages +include their SQLite support and embedded migrations; they do not require +`goose` or `sqlc` at runtime. Daemons still need configuration and a prepared +database; see the operator guide below. Use the full bundle for `dbl demo`, `dbl up`, `dbl dispatcher up`, `dbl executor up`, and bundled samples. The CLI-only package is for connecting to an existing dispatcher and submitting your own WASM files. diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 3a8ddb57..7c6cb459 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Download and install a versioned Debuglet release for Linux amd64. +# Verify and install an exact Debuglet release or local development package. set +x set -eu LC_ALL=C @@ -35,7 +35,7 @@ valid_version() ( ) version=${DEBUGLET_VERSION:-} -valid_version "$version" || fail 'set DEBUGLET_VERSION to a published version such as v1.2.3 or v1.2.3-rc.1' +valid_version "$version" || fail 'set DEBUGLET_VERSION to an exact package version, such as v1.2.3 or v0.0.0-dev.0123456789ab' prefix=${DEBUGLET_PREFIX:-${HOME:?HOME or DEBUGLET_PREFIX is required}/.local} carriage_return=$(printf '\r') case "$prefix" in ''|*' @@ -63,6 +63,12 @@ installer_name=install$suffix.sh checksums_name=SHA256SUMS$suffix release=https://github.com/netsec-ethz/debuglet/releases/download/$version release_dir=${DEBUGLET_RELEASE_DIR:-} +case "$version" in + v0.0.0-dev.*) + [ -n "$release_dir" ] || fail 'development candidates are not public releases; set DEBUGLET_RELEASE_DIR to the local package directory supplied by your operator' + [ "${DEBUGLET_ALLOW_UNSIGNED:-}" = 1 ] || fail 'development candidates are unsigned; set DEBUGLET_ALLOW_UNSIGNED=1 explicitly after obtaining the package through a trusted channel' + ;; +esac if [ -z "$release_dir" ]; then command -v curl >/dev/null 2>&1 || fail 'required command is missing: curl' else @@ -187,6 +193,10 @@ printf '\nInstalled Debuglet %s. Next commands:\n' "$version" printf 'export PATH=' shell_quote "$prefix/bin" printf ':"$PATH"\n' +if [ -z "$component" ] || [ "$component" = cli ]; then + shell_quote "$prefix/bin/dbl" + printf ' --output json version\n' +fi if [ -n "$component" ]; then printf 'Installed component: %s\n' "$component" exit 0 diff --git a/tools/test_bootstrap.py b/tools/test_bootstrap.py index 4efa15cb..8e77e54f 100644 --- a/tools/test_bootstrap.py +++ b/tools/test_bootstrap.py @@ -11,7 +11,7 @@ BOOTSTRAP = Path(__file__).resolve().parents[1] / 'scripts' / 'bootstrap.sh' -VERSION = 'v0.0.0-dev.0123456789ab' +VERSION = 'v1.2.3' ARCHIVE = f'debuglet-{VERSION}-linux-amd64.tar.gz' RELEASE_URL = 'https://github.com/netsec-ethz/debuglet/releases/download/' FAKE_CURL = r'''#!PYTHON @@ -87,11 +87,11 @@ def make_assets(self, archive=b'tiny fixture package', installer=INSTALLER.encod }.items(): (self.assets / name).write_bytes(data) - def run_bootstrap(self, extra=None, trace=False): + def run_bootstrap(self, extra=None, trace=False, script=BOOTSTRAP): env = dict(self.env) if extra: env.update(extra) - args = ['/bin/sh'] + (['-x'] if trace else []) + [str(BOOTSTRAP)] + args = ['/bin/sh'] + (['-x'] if trace else []) + [str(script)] result = subprocess.run(args, cwd=self.root, env=env, capture_output=True, text=True, timeout=8, check=False) self.assertEqual(list(self.tmp.iterdir()), [], 'bootstrap left temporary files') @@ -124,11 +124,46 @@ def test_pinned_release_installs_and_prints_usable_commands(self): self.assertEqual(args[-4:], ['--version', VERSION, '--prefix', str(self.prefix)]) self.assertTrue((self.prefix / 'bin/dbl').is_file()) commands = result.stdout.split('Next commands:\n', 1)[1] + self.assertIn(' --output json version\n', commands) check = subprocess.run(['/bin/sh', '-c', commands + '\nprintf "%s\\n" "$PATH"'], env=self.env, text=True, capture_output=True, timeout=3, check=False) self.assertEqual(check.returncode, 0, check.stderr) self.assertTrue(check.stdout.startswith(str(self.prefix / 'bin') + ':')) + def test_standalone_development_candidate_requires_local_explicit_unsigned_install(self): + version = 'v0.0.0-dev.0123456789ab' + self.make_assets(version=version) + script = self.root / 'bootstrap.sh' + shutil.copyfile(BOOTSTRAP, script) + result = self.run_bootstrap({'DEBUGLET_VERSION': version}, script=script) + self.assert_not_installed(result) + self.assertIn('development candidates are not public releases', result.stderr) + self.assertFalse(self.curl_log.exists()) + + settings = {'DEBUGLET_VERSION': version, 'DEBUGLET_RELEASE_DIR': str(self.assets)} + result = self.run_bootstrap(dict(settings, DEBUGLET_ALLOW_UNSIGNED=''), script=script) + self.assert_not_installed(result) + self.assertIn('set DEBUGLET_ALLOW_UNSIGNED=1 explicitly', result.stderr) + self.assertFalse(self.curl_log.exists()) + + result = self.run_bootstrap(settings, script=script) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('unsigned legacy/development package', result.stderr) + self.assertFalse(self.curl_log.exists(), 'local candidate contacted downloader') + args = self.install_log.read_text().splitlines() + self.assertEqual(args[-4:], ['--version', version, '--prefix', str(self.prefix)]) + + def test_local_development_candidate_checks_bytes_before_installer(self): + version = 'v0.0.0-dev.0123456789ab' + self.make_assets(version=version) + with (self.assets / f'debuglet-{version}-linux-amd64.tar.gz').open('ab') as asset: + asset.write(b'changed after checksumming') + result = self.run_bootstrap({'DEBUGLET_VERSION': version, + 'DEBUGLET_RELEASE_DIR': str(self.assets)}) + self.assert_not_installed(result) + self.assertIn('checksum verification failed', result.stderr) + self.assertFalse(self.curl_log.exists()) + def test_valid_release_versions(self): for version in ('v1.2.3', 'v1.2.3-rc.1'): with self.subTest(version=version): From 890b5dc28f6b739b96164d93d2788ed1dc83bc20 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 10:21:50 +0200 Subject: [PATCH 3/5] Condense Linux installation routes --- README-install.md | 117 ++++++++++++++++------------------------------ 1 file changed, 40 insertions(+), 77 deletions(-) diff --git a/README-install.md b/README-install.md index 21b30e42..c3ee8195 100644 --- a/README-install.md +++ b/README-install.md @@ -1,34 +1,21 @@ # Install Debuglet -Use Linux amd64 and the exact package version recommended by your dispatcher -operator. For executor enrollment, managed services, and the local demo, choose -the **full bundle**. The CLI-only and daemon-only packages do not provide that -complete setup flow. - -The current executor setup flow requires a newer package than the published, -unsigned v0.2.0 release. Until a compatible signed release is published, your -operator must supply a private development candidate. A development candidate -is not a public release and has no GitHub release download URL. +Use Linux amd64, a POSIX shell, GNU tar, and coreutils. Choose the operator's +exact **full bundle** for executor enrollment, managed services, or the local +demo. No source checkout or compiler is needed. The published unsigned v0.2.0 +package predates `dbl executor join`; the current enrollment flow requires an +operator-supplied candidate until a compatible signed release is published. ## Install a private development candidate -No source checkout or compiler is required. Obtain these files, the exact -candidate version, and its full source revision through your operator's trusted -channel: - -- A trusted standalone copy of `bootstrap.sh` from the candidate source. -- `debuglet-VERSION-linux-amd64.tar.gz`, `install.sh`, and `SHA256SUMS`, kept - together in one local directory with their original filenames. - -The bootstrap verifies the archive and installer checksums **before executing -the installer**. The installer then verifies the package contents and installed -files. These checks detect changed bytes; an unsigned candidate does not -authenticate its publisher. Trust in both the bootstrap and the candidate comes -from the channel through which your operator supplies them. - -Set the exact values supplied by your operator. `VERSION_PROVIDED_BY_OPERATOR` -below is a placeholder for a version such as `v0.0.0-dev.0123456789ab`; do not -rename an archive or infer its version from an unrelated checkout. +Obtain a trusted standalone `bootstrap.sh`, the exact version and source +revision, and the matching `debuglet-VERSION-linux-amd64.tar.gz`, `install.sh`, +and `SHA256SUMS` through your operator's trusted channel. Keep those three +package files together with their original names. Replace the version +placeholder below with the supplied `v0.0.0-dev.` and use your +actual local paths. Candidates require a local directory and explicit unsigned +opt-in; they are not public releases. Checksums are verified before the installer +runs, but do not authenticate an unsigned publisher. ```sh DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR \ @@ -41,30 +28,22 @@ dbl --output json version dbl demo ``` -Use your actual local paths. Leave `DEBUGLET_COMPONENT` unset for the full -bundle. The command refuses development candidates without a local package -directory and explicit unsigned opt-in. Signature requirements or configured -release trust cannot be overridden by unsigned mode. - -Confirm that `dbl version` reports the supplied version and source revision. -The demo needs no service, account, wallet, or root access. Installing the -package does not enroll an executor or create a service account; continue with -[executor setup](docs/operations/executor-onboarding.md), or use -[managed services](docs/operations/services.md) for a system-wide installation. - -The installer requires a POSIX shell, GNU tar, and coreutils. Its versioned -files are under `PREFIX/lib/debuglet/VERSION`, and `PREFIX/bin/dbl` selects the -installed bundle. Installing another version does not restart an existing -service; follow the managed-service update procedure. +Leave component and signature/trust settings unset for this unsigned full +bundle; unsigned mode cannot override configured signature requirements. +Confirm the reported version and source revision. The demo needs no service, +account, wallet, or root access. Installation does not enroll a node, create a +service account, or restart an existing service; follow [executor +setup](docs/operations/executor-onboarding.md) or [managed +services](docs/operations/services.md) next. ## Install a signed release -Use this route only after your operator identifies an exact published version -with downloadable signed assets. Obtain the standalone bootstrap and provision -the release signer's trust file independently, as described in -[Signed releases and offline verification](docs/operations/releases.md). -This needs curl, Python 3.11 or later, and OpenSSH with `ssh-keygen -Y` support -in addition to the installer prerequisites. +Once your operator identifies an exact published version with signed assets, +obtain the trusted standalone bootstrap and provision the signer's trust file +independently using [Signed releases and offline +verification](docs/operations/releases.md). Replace the version placeholder +below and leave unsigned, component and local-directory settings unset. This +route also requires curl, Python 3.11 or later, and OpenSSH with `ssh-keygen -Y`. ```sh DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR \ @@ -75,16 +54,13 @@ DEBUGLET_PREFIX="$HOME/.local" \ sh /path/to/trusted/bootstrap.sh ``` -Leave `DEBUGLET_RELEASE_DIR` unset for public release downloads. For a signed -offline install, set it to the directory containing the selected package files, -`release.json`, and `release.json.sig`. The bootstrap verifies the signature -before executing the installer and never falls back to unsigned mode. +Signature verification precedes installer execution and never falls back to +unsigned mode. For offline use, set `DEBUGLET_RELEASE_DIR` to the package +directory including `release.json` and `release.json.sig`. -For the older public demo only, a trusted bootstrap can download the existing -unsigned package with `DEBUGLET_VERSION=v0.2.0 DEBUGLET_ALLOW_UNSIGNED=1 sh -/path/to/trusted/bootstrap.sh`. Leave the component, local-directory and -signature settings unset for that separate legacy invocation. v0.2.0 does not -provide `dbl executor join`. +For the older public v0.2.0 demo only, use `DEBUGLET_VERSION=v0.2.0 +DEBUGLET_ALLOW_UNSIGNED=1 sh /path/to/trusted/bootstrap.sh` with component, +local-directory and signature/trust settings unset. ## Supported platforms @@ -147,27 +123,14 @@ Component packages are available from current source builds and are intended for | Executor operators | `executor` | `.cache/ci/packages/executor/` | `debuglet-executor` | | Dispatcher operators | `dispatcher` | `.cache/ci/packages/dispatcher/` | `debuglet-dispatcher` | -Obtain the selected component's three files and the trusted bootstrap from -your operator, or use the source build directory above. Select the same exact -version as the full bundle: - -```sh -DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR \ -DEBUGLET_COMPONENT=cli \ -DEBUGLET_RELEASE_DIR="$HOME/Downloads/debuglet-candidate/cli" \ -DEBUGLET_ALLOW_UNSIGNED=1 \ -DEBUGLET_PREFIX="$HOME/.local" \ -sh "$HOME/Downloads/bootstrap.sh" -export PATH="$HOME/.local/bin:$PATH" -``` - -Replace `cli` with `executor` or `dispatcher` and use its matching directory -when needed. Each directory contains its archive, `install-COMPONENT.sh`, and -`SHA256SUMS-COMPONENT`. These filenames are also unique release asset names. -For signed components, use the signed-release command with -`DEBUGLET_COMPONENT` instead of unsigned mode. Role installations share a prefix -safely: upgrading one changes only its own command link. The daemon packages -include their SQLite support and embedded migrations; they do not require +Use either bootstrap invocation above with `DEBUGLET_COMPONENT=cli`, +`executor`, or `dispatcher`. For local packages, point `DEBUGLET_RELEASE_DIR` +to that component's directory containing its archive, `install-COMPONENT.sh`, +and `SHA256SUMS-COMPONENT`. Select the same exact version as the full bundle. +Check the installed command with `--help`; the component packages omit the +bundled demo. Role installations share a prefix safely: upgrading one changes +only its own command link. The daemon packages include their SQLite support +and embedded migrations; they do not require `goose` or `sqlc` at runtime. Daemons still need configuration and a prepared database; see the operator guide below. From 8870ee5e2fb4bae430c11dc1f171d6c4e0c0d4c6 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 10:22:25 +0200 Subject: [PATCH 4/5] Add explicit managed executor certificate renewal --- cmd/dbl/executor_join.go | 91 ++++++++++--- cmd/dbl/executor_join_renew.go | 48 +++++++ cmd/dbl/executor_join_renew_test.go | 180 +++++++++++++++++++++++++ docs/operations/executor-onboarding.md | 89 ++++++++++-- docs/operations/services.md | 15 +++ internal/demo/service/enrolled.go | 28 ++-- internal/demo/service/renew.go | 103 ++++++++++++++ 7 files changed, 516 insertions(+), 38 deletions(-) create mode 100644 cmd/dbl/executor_join_renew.go create mode 100644 cmd/dbl/executor_join_renew_test.go create mode 100644 internal/demo/service/renew.go diff --git a/cmd/dbl/executor_join.go b/cmd/dbl/executor_join.go index c1460e8f..f31d3b67 100644 --- a/cmd/dbl/executor_join.go +++ b/cmd/dbl/executor_join.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "crypto" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" @@ -13,6 +14,7 @@ import ( "encoding/json" "encoding/pem" "errors" + "flag" "fmt" "io" "net" @@ -35,6 +37,7 @@ import ( const executorJoinUsage = `Usage: dbl executor join --dispatcher URL --executor ID --state-dir DIR [--ca-file FILE] [--token-file FILE] [--capacity BITS_PER_SECOND] [--host-tag TAG ...] + [--renew] Enroll this machine using the one-time token from Console > My nodes. The token is read without echo from the terminal, or from standard input or @@ -43,13 +46,19 @@ listing: home, office, datacentre, academic, cloud, dsl, cable, fibre, wifi, mobile, satellite or nat; it is written to [metadata] host_tags. HTTPS verifies the dispatcher using the system trust store or --ca-file; plain HTTP is accepted only on a literal loopback IP. -DIR must not exist. The command creates private identity, configuration and +For a new enrollment DIR must not exist. The command creates private identity, configuration and storage files, then prints the command to start the executor. It does not start a service or configure payments. Keep DIR for subsequent restarts. --capacity sets the bandwidth offered to the dispatcher (default 10000000 bit/s). Choose a sustainable limit for this host and its uplink. For a persistent Linux system service, follow docs/operations/executor-onboarding.md and enroll directly into /var/lib/debuglet/executors/NAME before service install. + +--renew replaces only the certificate of an existing managed executor, using a +fresh Set up again token. First drain its service and confirm joined shutdown +and disabled boot startup. Its key, configuration, database and native trust +stay unchanged. Resume the service explicitly after success. See the renewal +procedure in docs/operations/executor-onboarding.md; renewal is not automatic. ` type executorJoinResponse struct { @@ -61,10 +70,15 @@ type executorJoinResponse struct { } func executorJoinCommand(ctx context.Context, args []string, options globalOptions, stdout, stderr io.Writer) int { + return executorJoinCommandWith(ctx, args, options, stdout, stderr, productionServiceDependencies()) +} + +func executorJoinCommandWith(ctx context.Context, args []string, options globalOptions, stdout, stderr io.Writer, deps serviceDependencies) int { fs := newCommandFlagSet("executor join") endpoint := fs.String("dispatcher", "", "dispatcher HTTPS URL, including any API prefix") id := fs.String("executor", "", "executor ID shown in the console") - state := fs.String("state-dir", "", "new directory for this executor's identity and storage") + state := fs.String("state-dir", "", "directory for this executor's identity and storage") + renew := fs.Bool("renew", false, "renew a drained managed executor certificate using a fresh setup token") caFile := fs.String("ca-file", "", "trusted dispatcher CA certificate") tokenFile := fs.String("token-file", "", "file containing the enrollment token") capacity := fs.Int64("capacity", 10_000_000, "bandwidth offered to the dispatcher in bits per second") @@ -77,7 +91,14 @@ func executorJoinCommand(ctx context.Context, args []string, options globalOptio return code } if fs.NArg() != 0 || options.EndpointSet || options.Dispatcher != "" || strings.TrimSpace(*state) == "" { - return usageError("dbl executor join", executorJoinUsage, stderr, "supply --dispatcher, --executor and a new --state-dir after executor join") + return usageError("dbl executor join", executorJoinUsage, stderr, "supply --dispatcher, --executor and --state-dir after executor join") + } + changedSettings := false + fs.Visit(func(f *flag.Flag) { + changedSettings = changedSettings || f.Name == "capacity" || f.Name == "host-tag" + }) + if *renew && changedSettings { + return usageError("dbl executor join", executorJoinUsage, stderr, "--renew preserves capacity and host tags; omit their flags") } if *capacity <= 0 { return usageError("dbl executor join", executorJoinUsage, stderr, "--capacity must be positive bits per second") @@ -108,7 +129,20 @@ func executorJoinCommand(ctx context.Context, args []string, options globalOptio transport.TLSClientConfig.RootCAs = roots } statePath, err := filepath.Abs(*state) - if err == nil { + var installer *service.Installer + var retained service.Profile + if err == nil && *renew { + var code int + var ok bool + installer, code, ok = newInstaller("dbl executor join", serviceOptions{}, deps, stderr) + if !ok { + return code + } + retained, err = installer.CheckCertificateRenewal(ctx, filepath.Base(statePath)) + if err == nil && (retained.StateDir != statePath || retained.ExecutorID != *id) { + err = errors.New("--renew must name the installed executor ID and its canonical managed state directory") + } + } else if err == nil { _, err = os.Lstat(statePath) if err == nil { err = fmt.Errorf("state directory already exists: %s; keep existing identities and choose a new directory", statePath) @@ -138,6 +172,17 @@ func executorJoinCommand(ctx context.Context, args []string, options globalOptio client := &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("enrollment redirects are not followed; use the dispatcher's HTTPS API URL") }} + if *renew { + if err := renewExecutorCertificate(ctx, installer, retained, client, joinURL, token); err != nil { + return reportFailure(ctx, "dbl executor join", stderr, err) + } + return emitReported(ctx, "dbl executor join", options.Output, stdout, stderr, map[string]string{ + "executor_id": *id, "state_dir": statePath, "operation": "renewed", + }, func(w io.Writer) error { + _, err := fmt.Fprintf(w, "Executor %s certificate renewed. Key, configuration and stored work retained.\nThe service remains stopped and disabled. Resume explicitly:\n sudo %s drain --role executor --name %s --resume\n", *id, roleShellWord(executable), roleShellWord(retained.Name)) + return err + }) + } if err := joinExecutor(ctx, client, joinURL, *id, statePath, token, hostTags, *capacity); err != nil { return reportFailure(ctx, "dbl executor join", stderr, err) } @@ -302,38 +347,51 @@ func joinExecutor(ctx context.Context, client *http.Client, endpoint, id, state, if err := storagecheck.BootstrapFresh(ctx, storagecheck.Executor, filepath.Join(state, "executor.sqlite")); err != nil { return err } - csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{Subject: pkix.Name{CommonName: id}}, key) + result, err := requestExecutorCertificate(ctx, client, endpoint, id, token, key, keyPEM) if err != nil { return err } + for _, file := range []struct{ name, data string }{{"executor.crt", result.CertificatePEM}, {"ca.crt", result.CAPEM}, {"service.toml", executorJoinConfig(id, state, result, hostTags, capacity)}} { + if err := writeJoinFile(filepath.Join(state, file.name), []byte(file.data)); err != nil { + return err + } + } + return nil +} + +func requestExecutorCertificate(ctx context.Context, client *http.Client, endpoint, id, token string, key crypto.Signer, keyPEM []byte) (executorJoinResponse, error) { + var result executorJoinResponse + csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{Subject: pkix.Name{CommonName: id}}, key) + if err != nil { + return result, err + } body, err := json.Marshal(map[string]string{"executor_id": id, "token": token, "csr": string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csr}))}) if err != nil { - return err + return result, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) if err != nil { - return err + return result, err } req.Header.Set("Content-Type", "application/json") req.Header.Set("Debuglet-API-Version", "1.10") resp, err := client.Do(req) if err != nil { - return fmt.Errorf("request executor enrollment: %w", err) + return result, fmt.Errorf("request executor enrollment: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return fmt.Errorf("dispatcher refused enrollment (HTTP %d); check the dispatcher URL and replace expired or used tokens in My nodes", resp.StatusCode) + return result, fmt.Errorf("dispatcher refused enrollment (HTTP %d); check the dispatcher URL and replace expired or used tokens in My nodes", resp.StatusCode) } - var result executorJoinResponse decoder := json.NewDecoder(io.LimitReader(resp.Body, 64*1024)) if err := decoder.Decode(&result); err != nil { - return fmt.Errorf("decode enrollment response: %w", err) + return result, fmt.Errorf("decode enrollment response: %w", err) } if decoder.Decode(new(any)) != io.EOF { - return errors.New("enrollment response contains unexpected data") + return result, errors.New("enrollment response contains unexpected data") } if err := validateEnrollmentCertificate(result, id, keyPEM); err != nil { - return err + return result, err } // The server can use a public certificate while issuing client identities // from its own CA. Keep both authenticated trust roots for control TLS. @@ -344,12 +402,7 @@ func joinExecutor(ctx context.Context, client *http.Client, endpoint, id, state, } } } - for _, file := range []struct{ name, data string }{{"executor.crt", result.CertificatePEM}, {"ca.crt", result.CAPEM}, {"service.toml", executorJoinConfig(id, state, result, hostTags, capacity)}} { - if err := writeJoinFile(filepath.Join(state, file.name), []byte(file.data)); err != nil { - return err - } - } - return nil + return result, nil } func validateEnrollmentCertificate(result executorJoinResponse, id string, keyPEM []byte) error { diff --git a/cmd/dbl/executor_join_renew.go b/cmd/dbl/executor_join_renew.go new file mode 100644 index 00000000..1c61568e --- /dev/null +++ b/cmd/dbl/executor_join_renew.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package main + +import ( + "context" + "crypto" + "crypto/tls" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + + "github.com/netsec-ethz/debuglet/internal/demo/service" +) + +func renewExecutorCertificate(ctx context.Context, installer *service.Installer, p service.Profile, client *http.Client, endpoint, token string) (err error) { + keyPEM, err := os.ReadFile(filepath.Join(p.StateDir, "executor.key")) + if err != nil { + return err + } + pair, err := tls.LoadX509KeyPair(filepath.Join(p.StateDir, "executor.crt"), filepath.Join(p.StateDir, "executor.key")) + if err != nil { + return err + } + key, ok := pair.PrivateKey.(crypto.Signer) + if !ok { + return errors.New("retained executor key cannot sign a certificate request") + } + defer func() { + if err != nil { + err = fmt.Errorf("%w; keep the service stopped. If enrollment was sent, obtain a fresh Set up again token and repeat the same --renew command; do not delete or re-enroll the state directory", err) + } + }() + result, err := requestExecutorCertificate(ctx, client, endpoint, p.ExecutorID, token, key, keyPEM) + if err != nil { + return err + } + if result.GRPCAddress != p.DispatcherGRPC || result.YamuxAddress != p.DispatcherHTTP { + return errors.New("renewal returned different native endpoints; ask the service operator for recovery instructions") + } + if err := installer.RenewCertificate(ctx, p.Name, []byte(result.CertificatePEM)); err != nil { + return fmt.Errorf("publish renewed certificate under retained native trust: %w", err) + } + return nil +} diff --git a/cmd/dbl/executor_join_renew_test.go b/cmd/dbl/executor_join_renew_test.go new file mode 100644 index 00000000..eb0c3073 --- /dev/null +++ b/cmd/dbl/executor_join_renew_test.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package main + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "math/big" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/netsec-ethz/debuglet/internal/demo/service" + "github.com/netsec-ethz/debuglet/internal/storagecheck" +) + +type renewalManager struct { + *recordingManager + failed bool +} + +func (m *renewalManager) State(ctx context.Context, unit string) (service.UnitState, error) { + state, err := m.recordingManager.State(ctx, unit) + state.Result = "success" + if m.failed { + state.Active, state.Result, state.ExecMainStatus = "failed", "exit-code", 1 + } + return state, err +} + +func TestExecutorJoinRenewManagedCertificate(t *testing.T) { + for _, scenario := range []string{"success", "expired certificate", "write failure retry", "changed endpoints", "changed issuer", "invalid response", "enabled unit", "running unit", "failed shutdown", "changed identity", "changed settings"} { + t.Run(scenario, func(t *testing.T) { + alterResponse := false + fixture := newJoinFixture(t, func(result *executorJoinResponse) { + if alterResponse && scenario == "changed endpoints" { + result.GRPCAddress = "another.example:9001" + } + if alterResponse && scenario == "invalid response" { + result.CertificatePEM = "not a certificate" + } + }) + root := t.TempDir() + state := service.StateDirectory(root, storagecheck.Executor, "worker") + var stdout, stderr bytes.Buffer + if code := run(t.Context(), fixture.args(state), &stdout, &stderr); code != exitOK { + t.Fatalf("join: %d %s", code, stderr.String()) + } + manager := newRecordingManager() + deps := serviceTestDependencies(t, manager, root) + observed := &renewalManager{recordingManager: manager} + deps.manager = func(string) (service.Manager, error) { return observed, nil } + deps.lookup = func(string, string) (service.Account, error) { return service.Account{UID: 4242, GID: 4343}, nil } + if code := serviceCommandWith(t.Context(), []string{"install", "--role", "executor", "--enrolled-state", state, "--start=false", "--enable=false"}, globalOptions{}, &stdout, &stderr, deps); code != exitOK { + t.Fatalf("install: %d %s", code, stderr.String()) + } + certPath := filepath.Join(state, "executor.crt") + if err := os.Chmod(certPath, 0640); err != nil { + t.Fatal(err) + } + if scenario == "expired certificate" { + pair, err := tls.LoadX509KeyPair(certPath, filepath.Join(state, "executor.key")) + if err != nil { + t.Fatal(err) + } + key := pair.PrivateKey.(crypto.Signer) + template := &x509.Certificate{SerialNumber: big.NewInt(99), Subject: pkix.Name{CommonName: joinExecutorID}, NotBefore: time.Now().Add(-2 * time.Hour), NotAfter: time.Now().Add(-time.Hour)} + der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0640); err != nil { + t.Fatal(err) + } + } + if scenario == "changed issuer" { + // The API remains trusted, but its issuer is absent from the saved + // native roots. Renewal may not silently replace those roots. + data, _ := os.ReadFile(fixture.caFile) + if err := os.WriteFile(filepath.Join(state, "ca.crt"), data, 0600); err != nil { + t.Fatal(err) + } + } + original := map[string][]byte{} + for _, name := range []string{"executor.key", "executor.crt", "ca.crt", "executor.sqlite", "service.toml"} { + data, err := os.ReadFile(filepath.Join(state, name)) + if err != nil { + t.Fatal(err) + } + original[name] = data + } + info, _ := os.Stat(certPath) + owner := info.Sys().(*syscall.Stat_t) + chowned := false + failWrite := scenario == "write failure retry" + deps.chown = func(_ *os.File, uid, gid int) error { + if failWrite { + return errors.New("fixture refused certificate ownership") + } + if uid != int(owner.Uid) || gid != int(owner.Gid) { + t.Errorf("certificate owner changed: %d:%d", uid, gid) + } + chowned = true + return nil + } + args := append(fixture.args(state)[2:], "--renew") + unit := "debuglet-executor-worker.service" + switch scenario { + case "enabled unit": + manager.enabled[unit] = true + case "running unit": + manager.mainPID = 12345 + case "failed shutdown": + observed.failed = true + case "changed identity": + for n, arg := range args { + if arg == joinExecutorID { + args[n] = "00000000-1111-2222-3333-444444444444" + } + } + case "changed settings": + args = append(args, "--capacity", "1000") + } + alterResponse = true + renew := func() int { + stdout.Reset() + stderr.Reset() + return executorJoinCommandWith(context.Background(), args, globalOptions{}, &stdout, &stderr, deps) + } + code := renew() + wantSuccess := scenario == "success" || scenario == "expired certificate" + if !wantSuccess { + if code == exitOK || strings.Contains(stdout.String(), "certificate renewed") { + t.Fatalf("renewal unexpectedly succeeded: %s %s", stdout.String(), stderr.String()) + } + for name, want := range original { + got, _ := os.ReadFile(filepath.Join(state, name)) + if !bytes.Equal(got, want) { + t.Fatalf("failed renewal changed %s", name) + } + } + if scenario != "write failure retry" { + return + } + if !strings.Contains(stderr.String(), "fresh Set up again token") { + t.Fatalf("missing recovery instructions: %s", stderr.String()) + } + failWrite = false + code = renew() + } + if code != exitOK || !strings.Contains(stdout.String(), "certificate renewed") || !chowned { + t.Fatalf("renewal: %d %s %s", code, stdout.String(), stderr.String()) + } + for name, want := range original { + got, _ := os.ReadFile(filepath.Join(state, name)) + if bytes.Equal(got, want) == (name == "executor.crt") { + t.Errorf("unexpected contents after renewal: %s", name) + } + } + info, _ = os.Stat(certPath) + if info.Mode().Perm() != 0640 || manager.enabled[unit] || manager.mainPID != 0 { + t.Fatal("renewal changed certificate mode or enabled/started unit") + } + if strings.Contains(stdout.String()+stderr.String(), fixSecret) { + t.Fatal("renewal disclosed token") + } + }) + } +} diff --git a/docs/operations/executor-onboarding.md b/docs/operations/executor-onboarding.md index f7bff11c..9de81be3 100644 --- a/docs/operations/executor-onboarding.md +++ b/docs/operations/executor-onboarding.md @@ -133,15 +133,18 @@ the running executor; retain its generated identity and database for later start ## Run persistently on Linux -For a Linux amd64 machine with systemd, download the full package, `install.sh` -and `SHA256SUMS` recommended by the service operator. Use this sequence for a -**new enrollment**, replacing the package version, dispatcher URL and executor -ID with the supplied values: +For a Linux amd64 machine with systemd, first install the operator-recommended +**full** bundle under `/usr/local` using the [installation guide](../../README-install.md). +The CLI-only bundle cannot run an executor. Verify the exact version with +`/usr/local/bin/dbl --output json version` before enrollment. For an unpublished +candidate, use the guide's trusted local package directory and explicit unsigned +candidate option; do not use an invented release URL. Package installation does +not create a service account or enroll a node. + +Use this sequence for a **new enrollment**, replacing the dispatcher URL and +executor ID with the supplied values: ```sh -DEBUGLET_VERSION=VERSION_PROVIDED_BY_OPERATOR -sudo sh ./install.sh --archive "./debuglet-${DEBUGLET_VERSION}-linux-amd64.tar.gz" \ - --checksums ./SHA256SUMS --version "$DEBUGLET_VERSION" --prefix /usr/local # Create this account once; omit useradd if debuglet already exists. sudo useradd --system --user-group --home-dir /var/lib/debuglet --shell /usr/sbin/nologin debuglet sudo install -d -m 0755 /var/lib/debuglet /var/lib/debuglet/executors @@ -215,7 +218,18 @@ account; do not obtain another setup token or purge the state to upgrade. Keep the generated state directory. Restarting with the same configuration, private key, certificate and database preserves the machine identity and does -not need another setup token. A stopped machine remains in My nodes. +not need another setup token. A stopped machine remains in My nodes. For a +managed service, perform a planned restart with: + +```sh +sudo dbl --output json drain --role executor --name worker +# Continue after joined: true and enabled: false. +sudo dbl drain --role executor --name worker --resume +sudo dbl service status --role executor --name worker +``` + +Check **Ready** in My nodes afterward. Do not use Set up again for an ordinary +restart or package upgrade. Setup tokens expire after 24 hours and are shown only when issued. If setup did not finish, use **Set up again** to get a new token. Issuing it invalidates any @@ -228,6 +242,59 @@ If a response was lost or writing the local state failed, obtain a fresh token and use a new private state directory. `join` refuses to overwrite an existing identity. Never put a setup token in a command argument, URL or shared log. +## Renew a managed executor's certificate + +Enrollment certificates last at most 90 days, bounded by the issuer's expiry. +Renewal is manual. Before expiry, use a full package with `executor join --renew` +and keep the same managed executor ID and state directory. This operation uses +a fresh **Set up again** token from the existing node; it retains the private key, +configuration, database, native trust, service account and installed package. +It replaces only `executor.crt`. Do not create another node or move its state. + +1. Drain the service and require `joined: true` and `enabled: false`: + + ```sh + sudo dbl --output json drain --role executor --name worker + ``` + +2. Keep other administrators, automation and foreground executors from starting + this instance during renewal. Take a private offline backup of its full state + directory. In My nodes, choose **Set up again** for this same executor, then + run the renewal command and paste the fresh token at the hidden prompt: + + ```sh + sudo /usr/local/bin/dbl executor join --renew \ + --dispatcher https://debuglet.example/api --executor YOUR-EXECUTOR-ID \ + --state-dir /var/lib/debuglet/executors/worker + ``` + + Add `--ca-file` only if the HTTPS enrollment API needs a private CA. A private + `--token-file` also works; remove it after success. Do not add capacity or host + tag flags: renewal preserves the existing configuration. + +3. After the CLI reports successful certificate renewal, resume and inspect it: + + ```sh + sudo dbl drain --role executor --name worker --resume + sudo dbl service status --role executor --name worker + ``` + +The service stays stopped and disabled until you resume it. An expired old +certificate can be renewed with this same procedure after a clean joined stop. +A failed or uncertain request may already have consumed the token and replaced +the dispatcher's binding. Keep the service stopped, get a fresh **Set up again** +token, and repeat the exact `--renew` command. Do not restore the old certificate +and assume its binding still works. The command preserves local state on refusal; +a filesystem error after atomic publication can leave the new certificate in +place, so an uncertain result still needs this retry procedure. + +The command refuses changed native endpoints, a new certificate outside the +retained native trust, a changed executor ID, a modified unit, or a service that +has not shut down cleanly. Ask the service operator for recovery if the issuer +or endpoints changed, the key is lost, or shutdown cannot be confirmed. This +renewal option requires an installed canonical managed service; foreground +trial directories are not automatically converted or renewed. + ## Enable enrollment on a dispatcher Self-service enrollment is disabled by default. It requires native TLS with @@ -281,9 +348,9 @@ do not copy a root CA key or a server private key into the console. The browser never receives an issuer key or an executor private key. Enrollment issues client authentication certificates for at most 90 days, -bounded by issuer expiry. Renew a machine with **Set up again** before its -certificate expires; automatic certificate renewal is not provided. Existing -host-administered enrollment remains available. +bounded by issuer expiry. Use the [managed renewal procedure](#renew-a-managed-executors-certificate) +before its certificate expires; automatic certificate renewal is not provided. +Existing host-administered enrollment remains available. The new routes require API 1.10. Each account can create at most ten executors; their names and inventory are scoped to that account. Existing executors enrolled diff --git a/docs/operations/services.md b/docs/operations/services.md index d577b15d..125a990e 100644 --- a/docs/operations/services.md +++ b/docs/operations/services.md @@ -63,6 +63,21 @@ the capability-free fallback configuration; see [executor setup](executor-onboar Reference unit files live in [`deploy/systemd`](../../deploy/systemd). +## Restart or renew an enrolled executor + +For a planned restart, run `sudo dbl --output json drain --role executor --name +worker`, confirm `joined: true` and `enabled: false`, then run `sudo dbl drain +--role executor --name worker --resume`. Inspect `sudo dbl service status --role +executor --name worker` and the node's Ready state in the console. Reuse its +existing configuration and credentials; restart and package replacement do not +need a new setup token. + +Certificate renewal is a separate, manual operation using a fresh Set up again +token and `dbl executor join --renew`. It preserves the key and all stored work +while replacing the certificate of a stopped, disabled managed executor. Follow +the [renewal procedure](executor-onboarding.md#renew-a-managed-executors-certificate) +before the current certificate expires. + ## Replace an enrolled executor's package This procedure changes only the package used by an existing enrolled managed diff --git a/internal/demo/service/enrolled.go b/internal/demo/service/enrolled.go index a80f33e0..872715cc 100644 --- a/internal/demo/service/enrolled.go +++ b/internal/demo/service/enrolled.go @@ -53,9 +53,9 @@ func (i *Installer) prepareEnrolledState(ctx context.Context, p *Profile, accoun return writeRecord(*p) } -// validateEnrolledState reads the saved identity and trust without changing +// readEnrolledConfig reads the saved identity paths without changing // ownership, configuration, schema or the administrator's installed record. -func validateEnrolledState(p Profile) (*executorconfig.ExecutorConfig, error) { +func readEnrolledConfig(p Profile) (*executorconfig.ExecutorConfig, error) { for path := p.StateDir; path != p.Root && path != filepath.Dir(path); path = filepath.Dir(path) { info, err := os.Lstat(path) if err != nil || !info.IsDir() { @@ -87,29 +87,41 @@ func validateEnrolledState(p Profile) (*executorconfig.ExecutorConfig, error) { cfg.Credentials.ClientKey != filepath.Join(p.StateDir, "executor.key") || cfg.Credentials.EnrollmentToken != "" { return nil, errors.New("enrolled configuration must retain TLS and its original credentials and database inside the managed directory") } - pair, err := tlsfiles.KeyPair("credentials.client_cert", cfg.Credentials.ClientCert, "credentials.client_key", cfg.Credentials.ClientKey, time.Now()) + return cfg, nil +} + +func validateEnrolledState(p Profile) (*executorconfig.ExecutorConfig, error) { + cfg, err := readEnrolledConfig(p) if err != nil { return nil, err } + return cfg, validateEnrolledCertificate(cfg, cfg.Credentials.ClientCert) +} + +func validateEnrolledCertificate(cfg *executorconfig.ExecutorConfig, certificatePath string) error { + pair, err := tlsfiles.KeyPair("credentials.client_cert", certificatePath, "credentials.client_key", cfg.Credentials.ClientKey, time.Now()) + if err != nil { + return err + } if pair.Leaf.IsCA || pair.Leaf.Subject.CommonName != cfg.Identity.ExecutorID { - return nil, errors.New("enrolled certificate does not identify this executor") + return errors.New("enrolled certificate does not identify this executor") } roots, err := tlsfiles.TrustRoots("credentials.ca_cert", cfg.Credentials.CACert, time.Now()) if err != nil { - return nil, err + return err } intermediates := x509.NewCertPool() for _, der := range pair.Certificate[1:] { certificate, err := x509.ParseCertificate(der) if err != nil { - return nil, err + return err } intermediates.AddCert(certificate) } if _, err := pair.Leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil { - return nil, fmt.Errorf("verify enrolled certificate: %w", err) + return fmt.Errorf("verify enrolled certificate: %w", err) } - return cfg, nil + return nil } // This observes already-running daemons; administrators must keep foreground diff --git a/internal/demo/service/renew.go b/internal/demo/service/renew.go new file mode 100644 index 00000000..73d6427e --- /dev/null +++ b/internal/demo/service/renew.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 ETH Zurich + +package service + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "os" + "path/filepath" + "syscall" + + "github.com/netsec-ethz/debuglet/internal/fsutil" + "github.com/netsec-ethz/debuglet/internal/storagecheck" +) + +// CheckCertificateRenewal requires an enrolled managed executor with a clean, +// disabled shutdown. An expired old certificate remains usable for recovery; +// enrollment is authorized by a fresh account-issued token, not that certificate. +// Administrators must exclude concurrent starts and state changes throughout. +func (i *Installer) CheckCertificateRenewal(ctx context.Context, name string) (Profile, error) { + p, _, err := i.load(ctx, "renew-certificate", storagecheck.Executor, name) + if err != nil { + return p, err + } + if !p.Enrolled || p.ExecutorID == "" { + return p, errors.New("certificate renewal requires an installed enrolled executor") + } + state, err := i.manager.State(ctx, p.Unit) + if err != nil { + return p, err + } + if !state.Loaded || !state.Joined() || state.MainPID != 0 || state.Enabled { + return p, errors.New("certificate renewal requires a disabled executor with successful joined shutdown; drain it first") + } + if _, err := os.Lstat(p.UnitPath); err != nil { + return p, err + } + if err := i.verifyRemoval(ctx, p); err != nil { + return p, err + } + cfg, err := readEnrolledConfig(p) + if err != nil { + return p, err + } + if cfg.Dispatcher.Addr != p.DispatcherGRPC || cfg.Dispatcher.YamuxAddr != p.DispatcherHTTP || + (cfg.Network.PacketCounter == "auto") != p.PacketCapabilities { + return p, errors.New("certificate renewal cannot change the recorded dispatcher or counter mode") + } + // Parse and match the existing key without rejecting an expired leaf. + if _, err := tls.LoadX509KeyPair(cfg.Credentials.ClientCert, cfg.Credentials.ClientKey); err != nil { + return p, fmt.Errorf("read retained executor identity: %w", err) + } + return p, checkEnrolledProcesses("/proc", p, p.ExecutorID, 0) +} + +// RenewCertificate atomically publishes only a certificate valid for the saved +// key and retained trust roots. State, key, configuration, unit and record stay +// intact. It rechecks shutdown after the enrollment request has completed. +func (i *Installer) RenewCertificate(ctx context.Context, name string, certificate []byte) error { + p, err := i.CheckCertificateRenewal(ctx, name) + if err != nil { + return err + } + path := filepath.Join(p.StateDir, "executor.crt") + info, err := os.Lstat(path) + if err != nil { + return err + } + owner, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("certificate renewal cannot inspect certificate ownership") + } + file, err := os.CreateTemp(p.StateDir, ".renew-") + if err != nil { + return err + } + defer os.Remove(file.Name()) + if _, err := file.Write(certificate); err != nil { + file.Close() + return err + } + err = errors.Join(i.chown(file, int(owner.Uid), int(owner.Gid)), file.Chmod(info.Mode().Perm()), file.Sync(), file.Close()) + if err != nil { + return err + } + cfg, err := readEnrolledConfig(p) + if err != nil { + return err + } + if err := validateEnrolledCertificate(cfg, file.Name()); err != nil { + return err + } + if _, err := i.CheckCertificateRenewal(ctx, name); err != nil { + return err + } + if err := os.Rename(file.Name(), path); err != nil { + return err + } + return fsutil.SyncDir(p.StateDir) +} From 6fd046718ce0fad1484b0f0bf7a300ac59505245 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 10:24:00 +0200 Subject: [PATCH 5/5] Select console with managed executor renewal instructions --- configs/release-compatibility.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/release-compatibility.json b/configs/release-compatibility.json index 43c2d4ae..cfc1343e 100644 --- a/configs/release-compatibility.json +++ b/configs/release-compatibility.json @@ -1,4 +1,4 @@ { "console_repository": "https://gitlab.inf.ethz.ch/OU-PERRIG/yimin/debuglet/debuglet-dashboard", - "console_revision": "49f6ff6248000081287418d9d2c1d3f4a5cdc2bd" + "console_revision": "5312f70a1ebd9e78630ac6ffc4ae67383dcb4e14" }