From 9b5f45f75c1bbc4afa7780a57a16c22f29185c77 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 13:56:49 +0200 Subject: [PATCH 01/10] Bind Debian packages to release inventory and package CI --- .github/workflows/ci.yml | 4 +- Makefile | 5 +- README-install.md | 36 +++++++- docs/operations/releases.md | 24 +++++- scripts/ci-github.sh | 4 + scripts/package.sh | 3 + tools/release.py | 20 +++-- tools/release_inventory.py | 147 +++++++++++++++++++++++++++++++- tools/test_ci_github.py | 18 ++++ tools/test_release.py | 63 +++++++++++--- tools/test_release_inventory.py | 103 ++++++++++++++++++++++ tools/test_release_retention.py | 14 +++ 12 files changed, 409 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bbcbd17..f2657aa5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: artifact-name: ${{ format('package-{0}-{1}', github.sha, github.run_attempt) }} needs: build runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -119,6 +119,8 @@ jobs: path: | .cache/ci/package-install.* .cache/ci/package-install-*.json + .cache/ci/debian-*.jsonl + .cache/ci/debian-*.txt .cache/ci/ci-image-evidence/ include-hidden-files: true if-no-files-found: error diff --git a/Makefile b/Makefile index ee33f168..21bc4b82 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ CI_RACE_TIMEOUT ?= 5m CI_RACE_BUDGET_SECONDS ?= 240 CI_RACE_PACKAGE_PARALLEL ?= 2 -.PHONY: ci-test ci-vet ci-fmt ci-race ci-build ci-kernel ci-package ci-demo ci-compatibility ci-local +.PHONY: ci-test ci-vet ci-fmt ci-race ci-build ci-kernel ci-package ci-debian ci-demo ci-compatibility ci-local ci-test: python3 -m unittest -v tools/test_make_targets.py tools/test_ci_github.py tools/test_verify_pcap.py tools/test_release.py tools/test_release_inventory.py tools/test_release_retention.py @@ -63,6 +63,9 @@ ci-build: ci-package: GO="$(GO)" CI_DIST="$(CI_DIST)" bash scripts/ci-package.sh +ci-debian: + bash scripts/ci-debian.sh + ci-demo: GO="$(GO)" bash scripts/ci-demo.sh diff --git a/README-install.md b/README-install.md index c3ee8195..a66aec1a 100644 --- a/README-install.md +++ b/README-install.md @@ -1,11 +1,34 @@ # Install Debuglet -Use Linux amd64, a POSIX shell, GNU tar, and coreutils. Choose the operator's +For tar installation, use Linux amd64, a POSIX shell, GNU tar, and coreutils. Choose the operator's exact **full bundle** for executor enrollment, managed services, or the local demo. No source checkout or compiler is needed. The published unsigned v0.2.0 package predates `dbl executor join`; the current enrollment flow requires an operator-supplied candidate until a compatible signed release is published. +## Install with apt on Ubuntu 24.04 amd64 + +Use this route when your operator supplies a published apt repository, its +OpenPGP key fingerprint and an exact Debian package version. Verify the key +through an independent trusted channel, then install the operator's keyring and +repository configuration with `Signed-By` scoped to that keyring. Follow the +complete [apt setup procedure](docs/operations/apt-repository.md); no official +live repository URL or production signing key is implied by this guide. + +```sh +sudo apt-get update +sudo apt-get install debuglet=DEBIAN_VERSION_PROVIDED_BY_OPERATOR +dbl --output json version +dbl demo +``` + +The `debuglet` package selects the matching full payload, including the CLI, +executor, dispatcher and bundled samples. Installation stages software; choose +account setup, executor enrollment and service activation explicitly. Upgrades +retain payloads referenced by managed services. Follow the [Debian lifecycle +procedure](docs/operations/debian.md) to drain, back up, replace and resume an +existing service. Tar installation remains available below. + ## Install a private development candidate Obtain a trusted standalone `bootstrap.sh`, the exact version and source @@ -89,7 +112,8 @@ not establish support for Docker Desktop or emulation there. ## Build from source -On Linux amd64 with Go 1.26.9, Git, Make, Bash, GNU tar, and coreutils: +On Linux amd64 with Go 1.26.9, Git, Make, Bash, GNU tar, coreutils, Python 3.11 +or later, and `dpkg-deb` (the `dpkg` package): ```sh git clone https://github.com/netsec-ethz/debuglet.git @@ -104,7 +128,13 @@ the development-candidate command above with that version, `DEBUGLET_RELEASE_DIR="$PWD/.cache/ci/packages"`, and `sh scripts/bootstrap.sh`. For another machine, copy the trusted bootstrap and the three package files; the source checkout stays on the build machine. The build also produces the -component packages below. +component packages below and `debuglet-VERSION-linux-amd64.deb`, +`debuglet-payload-VERSION-linux-amd64.deb`, `deb-packages.json` and +`SHA256SUMS-debian` under `.cache/ci/packages/debian/`. The Debian version drops +the leading `v`, changes a prerelease hyphen to `~`, and appends `-1`; for example +`v1.2.3-rc.1` becomes `1.2.3~rc.1-1`. Debian files wrap the same full tar payload. +Use `make ci-debian` on an Ubuntu 24.04 amd64 Docker host to rehearse signed apt +metadata, installation and tamper rejection with disposable test keys. The package output includes `compatibility.json` and `SHA256SUMS-compatibility`. This separate record identifies the core package, diff --git a/docs/operations/releases.md b/docs/operations/releases.md index d1b3db19..2cfd8839 100644 --- a/docs/operations/releases.md +++ b/docs/operations/releases.md @@ -115,6 +115,8 @@ run the installed acceptance checks against those package bytes: ```sh make ci-build ci-package +# On an Ubuntu 24.04 amd64 Docker host: +make ci-debian make ci-demo ci-compatibility ci-local ``` @@ -124,6 +126,7 @@ not replace that exact-source CI run. The installed checks establish: | Check | Evidence | | --- | --- | +| Signed apt repository, installed Debian payload, state retention and tamper rejection with disposable trust | `debian-prepare.jsonl`, `debian-check.jsonl`, `debian-image.txt` | | Full and component installation, identical rerun, installed version/source | `package-install.log`, `package-install.json`, `package-install-{cli,executor,dispatcher}.json` | | Account-owned enrollment through the installed `dbl executor join`, TLS registration, completed hello, both daemon restarts, retained identity/session/output and fresh work | `role-tests.json`: `TestInstalledEnrollment` | | Published v0.2.0 executor compatibility and a populated v0.2.0 upgrade, old-binary downgrade refusal and restoration of the saved old state | `role-tests.json`: `TestInstalledReleasedCompatibility`, `TestInstalledReleasedUpgrade` | @@ -156,7 +159,13 @@ must succeed for that exact source, version and run attempt, including the vulnerability gate. Then manually dispatch **Prepare signed release** on that same tag and supply its completed CI run ID. Preparation checks the existing environment protection and exact gate results, and uses the already-built -packages; it neither rebuilds nor executes their payloads. +packages; it neither rebuilds nor executes their payloads. New package builds +include the two Debian files, `deb-packages.json` and `SHA256SUMS-debian`. +Preparation checks the exact archive and manifest identity, Debian payload +contents, command links, control fields and maintainer-script source hashes +using `dpkg-deb` without installing or running those files. The signed inventory +and provenance cover all four Debian subjects. Complete historical tar-only +signed bundles remain auditable; a partial Debian set is always rejected. After environment approval, a separate signing job receives only the prepared files. It checks out no source and signs `release.json` with the OpenSSH @@ -203,11 +212,20 @@ The installation asset set is: - `compatibility.json`, `SHA256SUMS-compatibility`, `release.json` and `release.json.sig`. -That is sixteen files, with `VERSION` and `COMPONENT` replaced by their exact -values. The complete signed bundle additionally contains inventory, provenance, +For releases containing Debian packages, also publish +`debuglet-VERSION-linux-amd64.deb`, `debuglet-payload-VERSION-linux-amd64.deb`, +`deb-packages.json` and `SHA256SUMS-debian`. This is twenty installation files +for new releases and sixteen for historical tar-only releases, with `VERSION` +and `COMPONENT` replaced by their exact values. The complete signed bundle additionally contains inventory, provenance, notices and acceptance evidence; retain it in the approved private archive below. Publishing the installation subset does not satisfy complete retention. +Publish an apt repository only from the complete audited signed release using +[the apt publication procedure](apt-repository.md). SSH release signatures bind +the build artifacts; apt additionally requires independently provisioned +OpenPGP trust and signed repository metadata. The package CI lane uses disposable +keys to verify the apt path, and does not provision production signing trust. + Download each component's files again from the published release and use the trusted bootstrap with `DEBUGLET_REQUIRE_SIGNATURE=1` and independently provisioned trust. Verify the version/source and each component's documented diff --git a/scripts/ci-github.sh b/scripts/ci-github.sh index e129549f..a255663b 100755 --- a/scripts/ci-github.sh +++ b/scripts/ci-github.sh @@ -168,3 +168,7 @@ docker run --rm --init --pull=never --name "$name" \ if [[ $lane == kernel ]]; then bash scripts/ci-shared-workers.sh fi + +if [[ $lane == package ]]; then + bash scripts/ci-debian.sh +fi diff --git a/scripts/package.sh b/scripts/package.sh index 26660671..44483612 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -10,3 +10,6 @@ for component in cli executor dispatcher; do done "${GO:-go}" run -mod=readonly ./internal/packaging compatibility \ -dist "${CI_DIST:-.cache/ci/dist}" -out "$package_dir" + +# Debian packages retain the verified full payload and use the same build identity. +bash scripts/package-deb.sh --package-dir "$package_dir" diff --git a/tools/release.py b/tools/release.py index a9766d5d..317c20a9 100644 --- a/tools/release.py +++ b/tools/release.py @@ -62,29 +62,33 @@ def digest(path): return {'sha256': hashlib.file_digest(data, 'sha256').hexdigest(), 'bytes': path.stat().st_size} -def package_names(version): +def package_names(version, debian=False): names = {'compatibility.json', 'SHA256SUMS-compatibility'} for component in ('', 'cli', 'executor', 'dispatcher'): suffix = '-' + component if component else '' names.update({f'debuglet{suffix}-{version}-linux-amd64.tar.gz', f'install{suffix}.sh', f'SHA256SUMS{suffix}'}) + if debian: + names.update({'deb-packages.json', 'SHA256SUMS-debian', + f'debuglet-{version}-linux-amd64.deb', + f'debuglet-payload-{version}-linux-amd64.deb'}) return names def stage(packages, directory, version): if not VERSION.fullmatch(version): raise ValueError('invalid release version') - expected = package_names(version) found = {} for path in packages.rglob('*'): if path.is_symlink(): raise ValueError('linked package input') if path.is_file(): - if path.name not in expected or path.name in found: + if path.name in found: raise ValueError('unexpected or duplicate package input: ' + path.name) found[path.name] = path + expected = package_names(version, debian='deb-packages.json' in found) if set(found) != expected: - raise ValueError('package artifact is incomplete') + raise ValueError('package artifact is incomplete or contains unexpected inputs') directory.mkdir(mode=0o700) for name, source in found.items(): shutil.copyfile(source, directory / name) @@ -146,7 +150,7 @@ def validate_gates(gates, version, source, builder): def subject(directory, version, source, builder): identity(version, source, builder) - expected = package_names(version) | METADATA | EVIDENCE_FILES + expected = package_names(version, debian=(directory / 'deb-packages.json').exists()) | METADATA | EVIDENCE_FILES if {p.name for p in directory.iterdir()} != expected: raise ValueError('release subjects require the exact packages and complete metadata') validate_gates(read_json(directory / 'gates.json'), version, source, builder) @@ -164,7 +168,8 @@ def load_subject(directory, version, source=None, builder=None): identity(result['version'], result['source_sha'], result['builder']) if result['version'] != version or (source and result['source_sha'] != source) or (builder and result['builder'] != builder): raise ValueError('release source, version or builder does not match') - if set(result['files']) != package_names(version) | METADATA | EVIDENCE_FILES: + expected = package_names(version, debian='deb-packages.json' in result['files']) | METADATA | EVIDENCE_FILES + if set(result['files']) != expected: raise ValueError('release subject set is incomplete') if {path.name for path in directory.iterdir()} - set(result['files']) - {SUBJECT, SUBJECT + '.sig'}: raise ValueError('release directory contains unsigned extra files') @@ -283,9 +288,10 @@ def archive_release(repository, version): def download_release(repository, record, directory, version): - expected = package_names(version) | METADATA | EVIDENCE_FILES | {SUBJECT, SUBJECT + '.sig'} assets = record.get('assets', []) names = [a.get('name') for a in assets] + expected = (package_names(version, debian='deb-packages.json' in names) | METADATA | EVIDENCE_FILES + | {SUBJECT, SUBJECT + '.sig'}) missing = sorted(expected - set(names)) if missing: raise ValueError('missing release assets: ' + ', '.join(missing)) diff --git a/tools/release_inventory.py b/tools/release_inventory.py index 9f9e276c..72073e40 100644 --- a/tools/release_inventory.py +++ b/tools/release_inventory.py @@ -159,7 +159,7 @@ def collect(source_directory, dist, image_evidence, out, go='go'): require('{path} {version} {sum}'.format(**module) in sums, 'binary dependency missing from committed go.sum') components.append({'name': name, **hashed, **info}) - paths = set(SOURCE_INPUTS) + paths = set(SOURCE_INPUTS) | set(DEBIAN_INPUTS) for directory in ('ratelimit', 'tagger'): for path in (source_directory / 'internal/executor' / directory / 'ebpf').rglob('*'): if path.is_file() and path.suffix in ('.c', '.h', '.o', '.go'): @@ -291,6 +291,142 @@ def payloads(directory, version, source, builder, inputs): return result +# Kept separate from SOURCE_INPUTS so historical tar-only signed inventories +# retain their original schema and remain auditable. +DEBIAN_INPUTS = ('deploy/debian/package.py', 'deploy/debian/payload-preinst', + 'deploy/debian/payload_guard.py', 'deploy/debian/selector-postinst', + 'scripts/package-deb.sh', 'docs/operations/debian.md') + + +class BoundedPipeReader: + def __init__(self, stream): + self.stream, self.position = stream, 0 + + def read(self, size): + require(0 <= size <= MAX_ARCHIVE_BYTES - self.position, 'Debian tar stream exceeds limit') + data = self.stream.read(size) + self.position += len(data) + return data + + +def deb_members(path, control=False): + """Read package contents with dpkg-deb; never install or execute scripts.""" + require(path.is_file() and not path.is_symlink() and path.stat().st_size <= MAX_ARCHIVE_BYTES, + 'Debian package must be a bounded regular file') + option = '--ctrl-tarfile' if control else '--fsys-tarfile' + result, directories = {}, set() + process = subprocess.Popen(['dpkg-deb', option, str(path)], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + try: + with tarfile.open(fileobj=BoundedPipeReader(process.stdout), mode='r|') as archive: + count = 0 + for member in archive: + count += 1 + require(count <= 256 and member.uid == 0 and member.gid == 0, + 'Debian package members or ownership differ') + name = member.name.removeprefix('./').rstrip('/') + if name in ('', '.') and member.isdir(): + continue + require(name and PurePosixPath(name).as_posix() == name and not name.startswith('/') + and '..' not in PurePosixPath(name).parts and name not in result + and name not in directories, 'unsafe or duplicate Debian member') + require(member.mode == (0o755 if member.isdir() or member.issym() + or name.startswith('usr/bin/') or '/bin/' in name + or (control and name != 'control') else 0o644) + or (member.issym() and member.mode == 0o777), 'Debian member mode differs: ' + name) + if member.isdir(): + directories.add(name) + elif member.issym() and not control: + result[name] = {'link': member.linkname} + else: + require(member.isfile() and member.size <= (65536 if control else 256 * 1024 * 1024), + 'unsupported or oversized Debian member') + data = archive.extractfile(member).read() + result[name] = data if control else digest(data) + process.stdout.close() + require(process.wait(timeout=10) == 0, 'cannot read Debian package') + finally: + process.stdout.close() + if process.poll() is None: + process.kill() + process.wait() + parents = {p.as_posix() for name in result for p in PurePosixPath(name).parents if p.as_posix() != '.'} + require(directories <= parents, 'unexpected Debian directory') + return result + + +def debian_payloads(directory, version, source, inputs): + metadata = directory / 'deb-packages.json' + if not metadata.exists(): + require(not list(directory.glob('*.deb')) and not (directory / 'SHA256SUMS-debian').exists(), + 'Debian packages require complete assembly metadata') + return [] + record = read_json(metadata) + fields = {'schema_version', 'upstream_version', 'debian_version', 'architecture', 'source_sha', + 'payload_package', 'payload_root', 'files', 'archive', 'manifest_sha256'} + require(set(record) == fields and record['schema_version'] == 1 + and record['upstream_version'] == version and record['source_sha'] == source + and record['architecture'] == 'amd64' + and record['debian_version'] == version[1:].replace('-', '~', 1) + '-1', + 'Debian assembly identity differs') + archive_name = f'debuglet-{version}-linux-amd64.tar.gz' + archive_path = directory / archive_name + require(record['archive'] == {'name': archive_name, **digest(regular(archive_path))}, + 'Debian source archive differs') + with gzip.open(archive_path, 'rb') as stream, tarfile.open(fileobj=BoundedTarReader(stream), mode='r:') as archive: + manifest_bytes = archive.extractfile(MANIFEST).read() + manifest = decode(manifest_bytes) + identity = digest(manifest_bytes)['sha256'] + payload_package = 'debuglet-payload-' + identity[:24] + payload_root = 'usr/lib/debuglet/debian/' + identity + require(record['manifest_sha256'] == identity and record['payload_package'] == payload_package + and record['payload_root'] == '/' + payload_root, 'Debian payload identity differs') + names = {f'debuglet-{version}-linux-amd64.deb', f'debuglet-payload-{version}-linux-amd64.deb'} + require(isinstance(record['files'], list) and len(record['files']) == 2 + and {item['name'] for item in record['files']} == names + and {p.name for p in directory.glob('*.deb')} == names, 'Debian package set differs') + materials = {m['name']: {k: m[k] for k in ('sha256', 'bytes')} for m in inputs['materials']} + require(set(DEBIAN_INPUTS) <= materials.keys(), 'missing Debian assembly source inputs') + expected_files = {payload_root + '/' + name: hashed for name, hashed in manifest['files'].items()} + expected_files[payload_root + '/' + MANIFEST] = digest(manifest_bytes) + for item in record['files']: + name = item['name'] + path = directory / name + require(item == {'name': name, **digest(regular(path))}, 'Debian package checksum differs') + payload = name.startswith('debuglet-payload-') + package = payload_package if payload else 'debuglet' + control_files = deb_members(path, control=True) + require(set(control_files) == ({'control', 'preinst', 'prerm'} if payload else {'control', 'postinst'}), + 'Debian control member set differs') + control = control_files['control'].decode() + fields = dict(line.split(': ', 1) for line in control.splitlines()) + expected_control = {'Package': package, 'Version': record['debian_version'], 'Architecture': 'amd64', + 'Maintainer': 'Debuglet maintainers', + 'Section': 'net', 'Priority': 'optional', 'Homepage': REPOSITORY, + 'Depends': 'python3' if payload else f"{payload_package} (= {record['debian_version']}), adduser", + 'Description': 'immutable full Debuglet payload' if payload else 'Debuglet command selector and local setup'} + require(fields == expected_control and len(control.splitlines()) == len(fields), + 'Debian control identity or dependencies differ') + for script, material in ({'preinst': 'payload-preinst', 'prerm': 'payload_guard.py'} if payload + else {'postinst': 'selector-postinst'}).items(): + content = control_files[script].replace(('/' + payload_root).encode(), b'@PAYLOAD_ROOT@') + content = content.replace(payload_package.encode(), b'@PAYLOAD_PACKAGE@') + if payload: + content = content.replace(record['debian_version'].encode(), b'@DEBIAN_VERSION@') + require(digest(content) == materials['deploy/debian/' + material], 'Debian maintainer script source differs') + expected_data = dict(expected_files) if payload else { + 'usr/bin/' + command: {'link': '../lib/debuglet/debian/' + identity + '/bin/' + command} + for command in ('dbl', 'debuglet-dispatcher', 'debuglet-executor')} + expected_data['usr/share/doc/' + package + '/copyright'] = materials['LICENSE'] + expected_data['usr/share/doc/' + package + '/debian.md'] = materials['docs/operations/debian.md'] + require(deb_members(path) == expected_data, 'Debian payload differs from verified full archive') + checksums = ''.join(item['sha256'] + ' ' + item['name'] + '\n' + for item in sorted(record['files'], key=lambda item: item['name'])) + checksums += digest(regular(metadata))['sha256'] + ' deb-packages.json\n' + require(regular(directory / 'SHA256SUMS-debian') == checksums.encode(), 'Debian checksum index differs') + return sorted(names) + + def validate_inputs(inputs, version, source, builder): require(inputs['schema_version'] == 1, 'unknown build inputs schema') record, image = inputs['build_record'], inputs['build_image'] @@ -370,6 +506,11 @@ def documents(directory, version, source, builder, inputs): dependencies.append('module:' + key) edges.append({'ref': child, 'dependsOn': dependencies}) edges.append({'ref': ref, 'dependsOn': children}) + for name in debian_payloads(directory, version, source, inputs): + ref = 'debian:' + name + components.append(component(ref, 'application', name, version, digest(regular(directory / name)), + purpose='debian-installation')) + edges.append({'ref': ref, 'dependsOn': ['archive:' + f'debuglet-{version}-linux-amd64.tar.gz']}) for key, module in sorted(modules.items()): components.append(component('module:' + key, 'library', module['path'], module['version'], module_sum=module['sum'], purpose='runtime')) @@ -389,8 +530,8 @@ def documents(directory, version, source, builder, inputs): 'properties': properties(source_sha=source, builder=builder)}, 'components': components, 'dependencies': edges} subject_files = sorted(p for p in directory.iterdir() if p.is_file() and - (p.name.endswith(('.tar.gz', '.sh')) or p.name.startswith('SHA256SUMS') - or p.name == 'compatibility.json')) + (p.name.endswith(('.tar.gz', '.sh', '.deb')) or p.name.startswith('SHA256SUMS') + or p.name in ('compatibility.json', 'deb-packages.json'))) subjects = [{'name': p.name, 'digest': {'sha256': digest(regular(p))['sha256']}} for p in subject_files] subjects.append({'name': 'sbom.cdx.json', 'digest': {'sha256': digest(encoded(sbom))['sha256']}}) provenance = {'_type': 'https://in-toto.io/Statement/v1', 'subject': subjects, diff --git a/tools/test_ci_github.py b/tools/test_ci_github.py index 72cdee18..4695eca0 100644 --- a/tools/test_ci_github.py +++ b/tools/test_ci_github.py @@ -82,6 +82,10 @@ def setUp(self): (self.root / 'scripts/ci-shared-workers.sh').write_text('''#!/usr/bin/env bash printf '["shared-workers"]\\n' >> "$DOCKER_CALLS" exit "${SHARED_WORKER_RESULT:-0}" +''') + (self.root / 'scripts/ci-debian.sh').write_text('''#!/usr/bin/env bash +printf '["debian"]\\n' >> "$DOCKER_CALLS" +exit "${DEBIAN_RESULT:-0}" ''') subprocess.run(['git', 'init', '-q', str(self.root)], check=True) subprocess.run(['git', 'add', '.'], cwd=self.root, check=True) @@ -178,6 +182,20 @@ def test_container_receives_only_public_metadata_and_scoped_mounts(self): self.assertNotIn(['shared-workers'], self.arguments()) self.assertEqual(self.arguments()[-1][0:2], ['rm', '--force']) + def test_package_runs_debian_acceptance_after_assembly_and_preserves_failures(self): + for changes, expected, hook in (({}, 0, True), ({'DOCKER_RESULT': '42'}, 42, False), + ({'DEBIAN_RESULT': '44'}, 44, True)): + with self.subTest(changes=changes): + self.calls.unlink(missing_ok=True) + result = self.launch('package', **changes) + self.assertEqual(result.returncode, expected, result.stderr) + calls = self.arguments() + self.assertEqual(['debian'] in calls, hook) + if hook: + run_index = next(i for i, call in enumerate(calls) if call[0] == 'run') + self.assertEqual(calls[run_index + 1], ['debian']) + self.assertEqual(calls[-1][0:2], ['rm', '--force']) + def test_kernel_adds_only_required_capabilities(self): result = self.launch('kernel') self.assertEqual(result.returncode, 0, result.stderr) diff --git a/tools/test_release.py b/tools/test_release.py index 4b1baee6..aab3a12b 100644 --- a/tools/test_release.py +++ b/tools/test_release.py @@ -84,20 +84,41 @@ def test_signed_gate_metadata_cannot_change_source_or_builder(self): release.validate_gates(gates, VERSION, SOURCE, BUILDER) def test_stage_requires_exact_component_set_and_preserves_bytes(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - packages = root / 'packages' - packages.mkdir() - expected = release.package_names(VERSION) - for name in expected: - (packages / name).write_bytes(name.encode()) - release.stage(packages, root / 'release', VERSION) - self.assertEqual({p.name for p in (root / 'release').iterdir()}, expected) - for path in (root / 'release').iterdir(): - self.assertEqual(path.read_bytes(), path.name.encode()) - (packages / 'unexpected').write_text('never sign') - with self.assertRaises(ValueError): - release.stage(packages, root / 'bad', VERSION) + for debian in (False, True): + with self.subTest(debian=debian), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + packages = root / 'packages' + packages.mkdir() + expected = release.package_names(VERSION, debian=debian) + for name in expected: + (packages / name).write_bytes(name.encode()) + release.stage(packages, root / 'release', VERSION) + self.assertEqual({p.name for p in (root / 'release').iterdir()}, expected) + for path in (root / 'release').iterdir(): + self.assertEqual(path.read_bytes(), path.name.encode()) + (packages / 'unexpected').write_text('never sign') + with self.assertRaises(ValueError): + release.stage(packages, root / 'bad', VERSION) + + def test_partial_debian_sets_cannot_be_staged_or_loaded(self): + complete = release.package_names(VERSION, debian=True) + debian_files = complete - release.package_names(VERSION) + for missing in debian_files: + with self.subTest(missing=missing), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + packages = root / 'packages' + packages.mkdir() + for name in complete - {missing}: + (packages / name).write_bytes(name.encode()) + with self.assertRaises(ValueError): + release.stage(packages, root / 'release', VERSION) + for name in release.METADATA | release.EVIDENCE_FILES: + (packages / name).write_bytes(name.encode()) + subject = {'schema_version': 1, 'version': VERSION, 'source_sha': SOURCE, 'builder': BUILDER, + 'files': {p.name: release.digest(p) for p in packages.iterdir()}} + (packages / 'release.json').write_text(json.dumps(subject)) + with self.assertRaises(ValueError): + release.load_subject(packages, VERSION, SOURCE, BUILDER) def test_subject_checksums_and_source_are_bound(self): with tempfile.TemporaryDirectory() as tmp: @@ -120,6 +141,20 @@ def test_subject_checksums_and_source_are_bound(self): with self.assertRaises(ValueError): release.load_subject(root, VERSION) + def test_complete_debian_subjects_are_required_and_bound(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + expected = release.package_names(VERSION, debian=True) | release.METADATA | release.EVIDENCE_FILES + for name in expected: + (root / name).write_bytes(name.encode()) + subject = {'schema_version': 1, 'version': VERSION, 'source_sha': SOURCE, 'builder': BUILDER, + 'files': {name: release.digest(root / name) for name in expected}} + (root / 'release.json').write_text(json.dumps(subject)) + release.load_subject(root, VERSION, SOURCE, BUILDER) + (root / f'debuglet-{VERSION}-linux-amd64.deb').write_bytes(b'tampered') + with self.assertRaisesRegex(ValueError, 'checksum mismatch'): + release.load_subject(root, VERSION, SOURCE, BUILDER) + class SecretRangeTests(unittest.TestCase): def test_tag_creation_scans_history_and_ordinary_update_scans_the_range(self): diff --git a/tools/test_release_inventory.py b/tools/test_release_inventory.py index ec84030d..4dbf659b 100644 --- a/tools/test_release_inventory.py +++ b/tools/test_release_inventory.py @@ -2,12 +2,15 @@ import importlib.util import gzip import io +import shutil +import subprocess from pathlib import Path import tarfile import tempfile import unittest from unittest import mock +ROOT = Path(__file__).resolve().parents[1] SPEC = importlib.util.spec_from_file_location('inventory', Path(__file__).with_name('release_inventory.py')) inv = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(inv) @@ -85,6 +88,106 @@ def setUp(self): for name in (archive_name, installer)) (self.root / ('SHA256SUMS-' + role if role else 'SHA256SUMS')).write_text(checksums) + def debian(self): + spec = importlib.util.spec_from_file_location('debian_package', ROOT / 'deploy/debian/package.py') + package = importlib.util.module_from_spec(spec) + spec.loader.exec_module(package) + self.inputs['materials'] += [{'name': name, **inv.digest((ROOT / name).read_bytes())} + for name in inv.DEBIAN_INPUTS] + self.input_path.write_bytes(inv.encoded(self.inputs)) + archive_name = f'debuglet-{VERSION}-linux-amd64.tar.gz' + with tarfile.open(self.root / archive_name) as archive: + files = {member.name: archive.extractfile(member).read() for member in archive} + identity = inv.digest(files[inv.MANIFEST])['sha256'] + package_name = 'debuglet-payload-' + identity[:24] + payload_root = '/usr/lib/debuglet/debian/' + identity + deb_version = package.debian_version(VERSION) + for payload in (False, True): + name = package_name if payload else 'debuglet' + with tempfile.TemporaryDirectory() as temporary: + tree = Path(temporary) + if payload: + for path, data in files.items(): + target = tree / payload_root.lstrip('/') / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + target.chmod(0o755 if path.startswith('bin/') else 0o644) + for target, template in (('preinst', 'payload-preinst'), ('prerm', 'payload_guard.py')): + package.write(tree / 'DEBIAN' / target, + package.template(template, payload_root, package_name, deb_version), 0o755) + else: + for command in ('dbl', 'debuglet-dispatcher', 'debuglet-executor'): + target = tree / 'usr/bin' / command + target.parent.mkdir(parents=True, exist_ok=True) + target.symlink_to('../lib/debuglet/debian/' + identity + '/bin/' + command) + package.write(tree / 'DEBIAN/postinst', (ROOT / 'deploy/debian/selector-postinst').read_text(), 0o755) + package.write(tree / 'DEBIAN/control', package.control(name, deb_version, + 'immutable full Debuglet payload' if payload else 'Debuglet command selector and local setup', + 'python3' if payload else f'{package_name} (= {deb_version}), adduser')) + package.write(tree / 'usr/share/doc' / name / 'copyright', self.inputs['notices']['LICENSE']) + package.write(tree / 'usr/share/doc' / name / 'debian.md', (ROOT / 'docs/operations/debian.md').read_text()) + for directory in tree.rglob('*'): + if directory.is_dir() and not directory.is_symlink(): + directory.chmod(0o755) + output = self.root / f"debuglet-{'payload-' if payload else ''}{VERSION}-linux-amd64.deb" + subprocess.run(['dpkg-deb', '--root-owner-group', '-Zgzip', '--build', str(tree), str(output)], + check=True, stdout=subprocess.DEVNULL) + record = {'schema_version': 1, 'upstream_version': VERSION, 'debian_version': deb_version, + 'architecture': 'amd64', 'source_sha': SOURCE, 'payload_package': package_name, + 'payload_root': payload_root, 'manifest_sha256': identity, + 'archive': {'name': archive_name, **inv.digest((self.root / archive_name).read_bytes())}} + self.refresh_debian(record) + return record + + def refresh_debian(self, record): + record['files'] = [{'name': path.name, **inv.digest(path.read_bytes())} + for path in sorted(self.root.glob('*.deb'))] + metadata = self.root / 'deb-packages.json' + metadata.write_bytes(inv.encoded(record)) + checksums = ''.join(item['sha256'] + ' ' + item['name'] + '\n' for item in record['files']) + checksums += inv.digest(metadata.read_bytes())['sha256'] + ' deb-packages.json\n' + (self.root / 'SHA256SUMS-debian').write_text(checksums) + + @unittest.skipUnless(shutil.which('dpkg-deb'), 'Debian inventory checks require dpkg-deb') + def test_debian_inventory_binds_exact_payload_source_and_provenance(self): + record = self.debian() + self.generate() + self.verify() + provenance = inv.read_json(self.root / 'provenance.json') + self.assertEqual(len(provenance['subject']), 19) + self.assertTrue({'deb-packages.json', 'SHA256SUMS-debian'} <= {s['name'] for s in provenance['subject']}) + sbom = inv.read_json(self.root / 'sbom.cdx.json') + debian = [item for item in sbom['dependencies'] if item['ref'].startswith('debian:')] + self.assertEqual(len(debian), 2) + self.assertTrue(all(item['dependsOn'] == [f'archive:debuglet-{VERSION}-linux-amd64.tar.gz'] for item in debian)) + for field, changed in (('source_sha', 'b' * 40), ('manifest_sha256', 'b' * 64), + ('archive', dict(record['archive'], sha256='b' * 64))): + with self.subTest(field=field): + original = record[field] + record[field] = changed + self.refresh_debian(record) + with self.assertRaisesRegex(ValueError, 'Debian'): + self.generate() + record[field] = original + self.refresh_debian(record) + + @unittest.skipUnless(shutil.which('dpkg-deb'), 'Debian inventory checks require dpkg-deb') + def test_debian_rejects_substituted_payload_and_scripts_even_with_fresh_checksums(self): + record = self.debian() + path = self.root / f'debuglet-payload-{VERSION}-linux-amd64.deb' + original = path.read_bytes() + for target in (record['payload_root'].lstrip('/') + '/bin/dbl', 'DEBIAN/prerm'): + with self.subTest(target=target), tempfile.TemporaryDirectory() as temporary: + path.write_bytes(original) + subprocess.run(['dpkg-deb', '--raw-extract', str(path), temporary], check=True) + with (Path(temporary) / target).open('ab') as output: + output.write(b'changed bytes') + subprocess.run(['dpkg-deb', '--root-owner-group', '-Zgzip', '--build', temporary, str(path)], + check=True, stdout=subprocess.DEVNULL) + self.refresh_debian(record) + with self.assertRaisesRegex(ValueError, 'Debian (payload|maintainer script)'): + self.generate() + def generate(self): inv.generate(self.root, VERSION, SOURCE, BUILDER, self.input_path) diff --git a/tools/test_release_retention.py b/tools/test_release_retention.py index ea0b1851..0e71fe15 100644 --- a/tools/test_release_retention.py +++ b/tools/test_release_retention.py @@ -215,6 +215,20 @@ def test_private_draft_verified_then_published_without_touching_old_versions(sel self.assertTrue(json.loads(result.stdout)['already_present']) self.assertFalse(any('POST' in call or 'PATCH' in call or 'upload' in call for call in self.calls()[count:])) + @unittest.skipUnless(shutil.which('dpkg-deb'), 'Debian retention requires dpkg-deb') + def test_debian_promotion_preserves_complete_signed_set(self): + self.inventory.debian() + self.inventory.generate() + self.signed() + result = self.promote() + self.assertEqual(result.returncode, 0, result.stderr) + assets = {asset['name'] for asset in self.read_state()['releases'][0]['assets']} + self.assertEqual(assets, {path.name for path in self.directory.iterdir()}) + self.assertTrue(release.package_names(VERSION, debian=True) <= assets) + result = self.promote() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(json.loads(result.stdout)['already_present']) + def test_existing_version_conflict_refuses_without_writes(self): self.signed() result = self.promote(); self.assertEqual(result.returncode, 0, result.stderr) From c78758f177e52f6853f92ec684aeb8ecfbf6d685 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 13:53:01 +0200 Subject: [PATCH 02/10] Add immutable Debian payload packages and safe lifecycle guards --- deploy/debian/package.py | 160 ++++++++++++++++++++++++++++++++ deploy/debian/payload-preinst | 19 ++++ deploy/debian/payload_guard.py | 90 ++++++++++++++++++ deploy/debian/selector-postinst | 22 +++++ deploy/debian/test_package.py | 155 +++++++++++++++++++++++++++++++ docs/operations/debian.md | 137 +++++++++++++++++++++++++++ scripts/package-deb.sh | 4 + 7 files changed, 587 insertions(+) create mode 100755 deploy/debian/package.py create mode 100755 deploy/debian/payload-preinst create mode 100755 deploy/debian/payload_guard.py create mode 100755 deploy/debian/selector-postinst create mode 100644 deploy/debian/test_package.py create mode 100644 docs/operations/debian.md create mode 100755 scripts/package-deb.sh diff --git a/deploy/debian/package.py b/deploy/debian/package.py new file mode 100755 index 00000000..ac5ed274 --- /dev/null +++ b/deploy/debian/package.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Wrap the existing verified full tar payload in co-installable Debian packages.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +VERSION = re.compile(r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-([0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*))?\Z") + + +def debian_version(version): + if len(version) > 128 or not VERSION.fullmatch(version): + raise ValueError("invalid upstream version") + return version[1:].replace("-", "~", 1) + "-1" + + +def digest(path): + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def write(path, content, mode=0o644): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + path.chmod(mode) + + +def template(name, root, package, version): + return ((HERE / name).read_text().replace("@PAYLOAD_ROOT@", root) + .replace("@PAYLOAD_PACKAGE@", package).replace("@DEBIAN_VERSION@", version)) + + +def control(name, version, description, depends): + return (f"Package: {name}\nVersion: {version}\nArchitecture: amd64\n" + "Maintainer: Debuglet maintainers\n" + "Section: net\nPriority: optional\n" + "Homepage: https://github.com/netsec-ethz/debuglet\n" + f"Depends: {depends}\nDescription: {description}\n") + + +def assemble(archive, checksums, output): + if sys.platform != "linux" or os.uname().machine != "x86_64": + raise ValueError("Debian assembly requires Linux amd64") + installer = archive.parent / "install.sh" + for path in (archive, checksums, installer): + if path.is_symlink() or not path.is_file(): + raise ValueError(f"input must be a regular file: {path}") + sums = {} + for line in checksums.read_text().splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) ([^/]+)", line) + if not match or match[2] in sums: + raise ValueError("invalid or duplicate checksum entry") + sums[match[2]] = match[1] + if set(sums) != {archive.name, "install.sh"}: + raise ValueError("checksums must name exactly the full archive and install.sh") + for path in (archive, installer): + if digest(path) != sums[path.name]: + raise ValueError(f"checksum mismatch: {path.name}") + with tarfile.open(archive, "r:gz") as bundle: + member = bundle.getmember("share/debuglet/manifest.json") + if not member.isfile() or member.size > 65536: + raise ValueError("invalid full payload manifest") + manifest_bytes = bundle.extractfile(member).read() + manifest = json.loads(manifest_bytes) + upstream = manifest["version"] + version = debian_version(upstream) + if (manifest.get("component") or manifest.get("schema_version") != 1 + or manifest.get("dirty") is not False or manifest.get("goos") != "linux" + or manifest.get("goarch") != "amd64" + or not re.fullmatch(r"[0-9a-f]{40}", manifest.get("source_sha", "")) + or archive.name != f"debuglet-{upstream}-linux-amd64.tar.gz"): + raise ValueError("Debian assembly requires a clean full Linux amd64 payload") + identity = hashlib.sha256(manifest_bytes).hexdigest() + payload_package = "debuglet-payload-" + identity[:24] + payload_root = "/usr/lib/debuglet/debian/" + identity + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()): + raise ValueError("Debian output directory must be empty") + with tempfile.TemporaryDirectory(prefix="debuglet-deb-") as temporary: + work = Path(temporary) + # Use the existing installer to verify exact members, modes and all + # hashes. No Debian-specific payload variant or executable is built. + subprocess.run(["sh", str(installer), "--archive", str(archive), "--checksums", str(checksums), + "--version", upstream, "--prefix", str(work / "verified"), "--stage-only"], check=True) + verified = work / "verified/lib/debuglet" / upstream + if (verified / "share/debuglet/manifest.json").read_bytes() != manifest_bytes: + raise ValueError("manifest changed during staging") + payload, selector = work / "payload", work / "selector" + shutil.copytree(verified, payload / payload_root.lstrip("/")) + write(payload / "DEBIAN/control", control(payload_package, version, + "immutable full Debuglet payload", "python3")) + write(payload / "DEBIAN/preinst", template("payload-preinst", payload_root, payload_package, version), 0o755) + write(payload / "DEBIAN/prerm", template("payload_guard.py", payload_root, payload_package, version), 0o755) + write(selector / "DEBIAN/control", control("debuglet", version, + "Debuglet command selector and local setup", f"{payload_package} (= {version}), adduser")) + write(selector / "DEBIAN/postinst", (HERE / "selector-postinst").read_text(), 0o755) + for command in ("dbl", "debuglet-dispatcher", "debuglet-executor"): + link = selector / "usr/bin" / command + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to("../lib/debuglet/debian/" + identity + "/bin/" + command) + for tree, name in ((payload, payload_package), (selector, "debuglet")): + documentation = tree / "usr/share/doc" / name + write(documentation / "copyright", (REPO / "LICENSE").read_text()) + write(documentation / "debian.md", (REPO / "docs/operations/debian.md").read_text()) + # Stable timestamps/root ownership keep builds independent of the + # invoking user. gzip is supported by Ubuntu's dpkg without extras. + for path in tree.rglob("*"): + os.utime(path, (0, 0), follow_symlinks=False) + filename = "debuglet" if name == "debuglet" else "debuglet-payload" + package_file = output / f"{filename}-{upstream}-linux-amd64.deb" + subprocess.run(["dpkg-deb", "--root-owner-group", "-Zgzip", "-z6", "--build", str(tree), str(package_file)], + env={**os.environ, "SOURCE_DATE_EPOCH": "0"}, check=True) + packages = [{"name": path.name, "sha256": digest(path), "bytes": path.stat().st_size} + for path in sorted(output.glob("*.deb"))] + result = {"schema_version": 1, "upstream_version": upstream, "debian_version": version, + "architecture": "amd64", "source_sha": manifest["source_sha"], + "payload_package": payload_package, "payload_root": payload_root, "files": packages, + "manifest_sha256": identity, + "archive": {"name": archive.name, "sha256": sums[archive.name], "bytes": archive.stat().st_size}} + write(output / "deb-packages.json", json.dumps(result, indent=2) + "\n") + write(output / "SHA256SUMS-debian", "".join( + f"{digest(path)} {path.name}\n" for path in [*sorted(output.glob("*.deb")), output / "deb-packages.json"])) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package-dir", type=Path, default=Path(os.environ.get("CI_PACKAGE_DIR", ".cache/ci/packages"))) + parser.add_argument("--archive", type=Path) + parser.add_argument("--checksums", type=Path) + parser.add_argument("--out", type=Path) + args = parser.parse_args() + if bool(args.archive) != bool(args.checksums): + parser.error("--archive and --checksums must be supplied together") + if args.archive: + archive, checksums = args.archive, args.checksums + else: + archives = list(args.package_dir.glob("debuglet-v*-linux-amd64.tar.gz")) + if len(archives) != 1: + parser.error("--package-dir must contain exactly one full archive") + archive, checksums = archives[0], args.package_dir / "SHA256SUMS" + result = assemble(archive.absolute(), checksums.absolute(), (args.out or args.package_dir / "debian").absolute()) + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, KeyError, subprocess.CalledProcessError, tarfile.TarError) as error: + sys.exit(f"package-deb: {error}") diff --git a/deploy/debian/payload-preinst b/deploy/debian/payload-preinst new file mode 100755 index 00000000..e925744d --- /dev/null +++ b/deploy/debian/payload-preinst @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +case "$1" in + install|upgrade) + if [ "$1" = upgrade ] && [ "${2-}" != '@DEBIAN_VERSION@' ]; then + echo 'debuglet: different payload versions must use separately named packages; upgrade the debuglet selector instead' >&2 + exit 1 + fi + # dpkg may retry an interrupted unpack or reinstall these exact bytes. + # Do not follow a replaced directory into another tree during repair. + for path in /usr /usr/lib /usr/lib/debuglet /usr/lib/debuglet/debian \ + '@PAYLOAD_ROOT@' '@PAYLOAD_ROOT@/bin' '@PAYLOAD_ROOT@/share' '@PAYLOAD_ROOT@/share/debuglet'; do + if [ -L "$path" ] || { [ -e "$path" ] && [ ! -d "$path" ]; }; then + echo "debuglet: refusing payload installation through $path" >&2 + exit 1 + fi + done + ;; +esac diff --git a/deploy/debian/payload_guard.py b/deploy/debian/payload_guard.py new file mode 100755 index 00000000..17552f5e --- /dev/null +++ b/deploy/debian/payload_guard.py @@ -0,0 +1,90 @@ +#!/usr/bin/python3 +"""Read-only dpkg removal guard; no runtime import or service-manager dependency. + +Installed as the payload's prerm, outside its unchanged distribution manifest. +The constants below are filled by package.py. Keep the retained-record policy +consistent with internal/demo/service/prune.go. +""" + +import json +import os +from pathlib import Path +import stat +import sys + +PAYLOAD_ROOT = "@PAYLOAD_ROOT@" +PAYLOAD_PACKAGE = "@PAYLOAD_PACKAGE@" +DEBIAN_VERSION = "@DEBIAN_VERSION@" + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def inside(path, candidate): + return path == candidate or path.startswith(candidate + "/") + + +def check_removal(candidate, records=Path("/etc/debuglet/services"), + proc=Path("/proc"), bins=(Path("/usr/bin"), Path("/usr/local/bin"))): + """Refuse unknown state as well as known references, without changing it.""" + for path in (records, *records.parents): + if path.is_symlink(): + raise ValueError(f"cannot inspect service records through symlink {path}") + if records.exists() or records.is_symlink(): + if records.is_symlink() or not records.is_dir(): + raise ValueError(f"cannot inspect retained service records at {records}") + for path in sorted(records.glob("*.json")): + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_size > 4096: + raise ValueError(f"cannot inspect retained service record {path}") + record = json.loads(path.read_bytes(), object_pairs_hook=unique_object) + root = record.get("payload_root") if isinstance(record, dict) else None + if not isinstance(root, str) or not root.startswith("/"): + raise ValueError(f"invalid retained service record {path}") + if os.path.normpath(root) == candidate or os.path.realpath(root) == candidate: + raise ValueError( + f"retained service {path.name} references this payload. " + "Drain and back up the role, then explicitly replace its package " + "while stopped and disabled. Uninstall without --purge retains " + "this recovery reference; --purge also deletes that role's data") + for directory in bins: + if not directory.exists(): + continue + for path in directory.iterdir(): + if path.is_symlink() and inside(os.path.realpath(path), candidate): + raise ValueError(f"command {path} still selects this payload; upgrade or remove the debuglet selector first") + for process in proc.iterdir(): + if not process.name.isdecimal(): + continue + try: + executable = os.readlink(process / "exe") + except FileNotFoundError: + continue # exited process or kernel thread + if inside(executable, candidate): + raise ValueError(f"process {process.name} is executing this payload; stop it before removal") + + +def main(): + action = sys.argv[1] if len(sys.argv) > 1 else "" + if action == "upgrade": + if len(sys.argv) < 3 or sys.argv[2] != DEBIAN_VERSION: + raise ValueError("different payload versions must use separately named packages; upgrade the debuglet selector instead") + return # same-version repair restores the same signed payload bytes + if action not in ("remove", "deconfigure"): + return + check_removal(PAYLOAD_ROOT) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError) as error: + print(f"{PAYLOAD_PACKAGE}: removal refused: {error}. " + f"This payload and all state have been retained. See /usr/share/doc/{PAYLOAD_PACKAGE}/debian.md", file=sys.stderr) + sys.exit(1) diff --git a/deploy/debian/selector-postinst b/deploy/debian/selector-postinst new file mode 100755 index 00000000..7709cb2c --- /dev/null +++ b/deploy/debian/selector-postinst @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu +if [ "$1" != configure ]; then exit 0; fi +# Existing identities, directory ownership and modes are operator-controlled. +if ! getent passwd debuglet >/dev/null; then + if ! getent group debuglet >/dev/null; then addgroup --system debuglet; fi + adduser --system --ingroup debuglet --home /var/lib/debuglet \ + --no-create-home --shell /usr/sbin/nologin debuglet +fi +for path in /var/lib/debuglet /var/lib/debuglet/executors; do + if [ -L "$path" ] || { [ -e "$path" ] && [ ! -d "$path" ]; }; then + echo "debuglet: expected a real directory at $path; existing state was preserved" >&2 + exit 1 + fi + if [ ! -d "$path" ]; then + mkdir -m 0755 "$path" + if [ "$path" = /var/lib/debuglet/executors ]; then + chown debuglet:"$(id -gn debuglet)" "$path" + fi + fi +done +echo 'Debuglet software is staged. Enrollment and service activation are explicit; existing services keep their recorded payload.' diff --git a/deploy/debian/test_package.py b/deploy/debian/test_package.py new file mode 100644 index 00000000..e69bbf98 --- /dev/null +++ b/deploy/debian/test_package.py @@ -0,0 +1,155 @@ +"""Focused package assembly and read-only removal regressions (Linux only).""" + +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from unittest import mock + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, Path(__file__).parent / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +package = load("debian_package", "package.py") +guard = load("payload_guard", "payload_guard.py") + + +class Versions(unittest.TestCase): + def test_prerelease_and_dev_sort_before_final(self): + for upstream, expected in (("v1.2.3", "1.2.3-1"), ("v1.2.3-rc.1", "1.2.3~rc.1-1"), + ("v0.0.0-dev.abcdef012345", "0.0.0~dev.abcdef012345-1")): + self.assertEqual(package.debian_version(upstream), expected) + subprocess.run(["dpkg", "--compare-versions", package.debian_version("v1.2.3-rc.1"), + "lt", package.debian_version("v1.2.3")], check=True) + for invalid in ("1.2.3", "v01.2.3", "v1.2.3+dirty", "v1.2.3-../x", "v1.2.3-"): + with self.assertRaises(ValueError): + package.debian_version(invalid) + + +class Removal(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.records, self.proc, self.bin = [self.root / name for name in ("records", "proc", "bin")] + for path in (self.records, self.proc, self.bin): + path.mkdir() + self.candidate = "/usr/lib/debuglet/debian/" + "a" * 64 + + def check(self): + guard.check_removal(self.candidate, self.records, self.proc, (self.bin,)) + + def test_inactive_unreferenced_payload_can_be_removed(self): + self.check() + + def test_retained_record_pins_stopped_payload(self): + (self.records / "executor-worker.json").write_text(json.dumps({"payload_root": self.candidate})) + with self.assertRaisesRegex(ValueError, "retained service"): + self.check() + self.assertTrue((self.records / "executor-worker.json").exists()) + + def test_corrupt_and_unreadable_records_fail_closed(self): + record = self.records / "executor-worker.json" + for content in ("{", "[]", "{}", '{"payload_root":"/a","payload_root":"/b"}', " " * 4097): + record.write_text(content) + with self.assertRaises(ValueError): + self.check() + record.unlink() + record.symlink_to(self.root / "missing") + with self.assertRaises(ValueError): + self.check() + + def test_unknown_proc_error_fails_closed(self): + (self.proc / "12").mkdir() + (self.proc / "12/exe").write_text("not a proc link") + with self.assertRaises(OSError): + self.check() + + def test_executing_payload_and_command_alias_pin_it(self): + (self.proc / "123").mkdir() + (self.proc / "123/exe").symlink_to(self.candidate + "/bin/debuglet-executor") + with self.assertRaisesRegex(ValueError, "process 123"): + self.check() + (self.proc / "123/exe").unlink() + (self.bin / "old-dbl").symlink_to(self.candidate + "/bin/dbl") + with self.assertRaisesRegex(ValueError, "command"): + self.check() + + def test_unrelated_process_and_record_do_not_pin(self): + (self.records / "executor-other.json").write_text(json.dumps({"payload_root": "/usr/local/lib/debuglet/v1.0.0"})) + (self.proc / "456").mkdir() + (self.proc / "456/exe").symlink_to("/usr/bin/python3") + self.check() + + def test_same_version_reinstall_does_not_remove_retained_payload(self): + with mock.patch.object(guard, "DEBIAN_VERSION", "1.2.3-1"): + with mock.patch.object(guard.sys, "argv", ["prerm", "upgrade", "1.2.3-1"]): + guard.main() + with mock.patch.object(guard.sys, "argv", ["prerm", "upgrade", "1.2.4-1"]): + with self.assertRaisesRegex(ValueError, "separately named"): + guard.main() + + +class Assembly(unittest.TestCase): + def test_full_payload_manifest_and_modes_are_unchanged(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + upstream = "v1.2.3-rc.1" + modes = {"LICENSE": 0o644, "README-install.md": 0o644, + "bin/dbl": 0o755, "bin/debuglet-executor": 0o755, "bin/debuglet-dispatcher": 0o755, + "share/debuglet/demo.wasm": 0o644, "share/debuglet/hello.wasm": 0o644} + contents = {name: ("fixture " + name).encode() for name in modes} + manifest = {"schema_version": 1, "version": upstream, "source_sha": "a" * 40, + "dirty": False, "go_version": "go1.26.9", "goos": "linux", "goarch": "amd64", + "guest_abi": "debuglet-go-wasi-imports-v1", "build_pipeline_url": "", + "files": {name: {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} + for name, data in contents.items()}} + contents["share/debuglet/manifest.json"] = json.dumps(manifest).encode() + modes["share/debuglet/manifest.json"] = 0o644 + archive = root / f"debuglet-{upstream}-linux-amd64.tar.gz" + with tarfile.open(archive, "w:gz") as bundle: + for name, data in sorted(contents.items()): + member = tarfile.TarInfo(name) + member.mode, member.size = modes[name], len(data) + bundle.addfile(member, io.BytesIO(data)) + installer = (package.REPO / "scripts/install.sh").read_text() + sums = "\n".join(f"{hashlib.sha256(data).hexdigest()} {name}" for name, data in sorted(contents.items())) + installer = installer.replace("@VERSION@", upstream).replace("@SOURCE_SHA@", "a" * 40).replace("@COMPONENT@", "").replace("@PAYLOAD_SUMS@", sums) + (root / "install.sh").write_text(installer) + checksums = root / "SHA256SUMS" + checksums.write_text("".join(f"{package.digest(path)} {path.name}\n" for path in (archive, root / "install.sh"))) + output = root / "debs" + result = package.assemble(archive, checksums, output) + payload = output / f"debuglet-payload-{upstream}-linux-amd64.deb" + extracted = root / "extracted" + subprocess.run(["dpkg-deb", "--extract", str(payload), str(extracted)], check=True) + installed = extracted / result["payload_root"].lstrip("/") + for name, data in contents.items(): + self.assertEqual((installed / name).read_bytes(), data) + self.assertEqual((installed / name).stat().st_mode & 0o777, modes[name]) + # Existing prune requires prefix/lib/debuglet/. + # A Debian hash root has no such valid version path component. + self.assertIsNone(package.VERSION.fullmatch(installed.name)) + selector = output / f"debuglet-{upstream}-linux-amd64.deb" + subprocess.run(["dpkg-deb", "--extract", str(selector), str(root / "selector")], check=True) + self.assertEqual(os.readlink(root / "selector/usr/bin/dbl"), + "../lib/debuglet/debian/" + installed.name + "/bin/dbl") + self.assertIn(result["payload_package"], subprocess.check_output(["dpkg-deb", "--field", str(selector), "Depends"], text=True)) + subprocess.run(["sha256sum", "--check", "SHA256SUMS-debian"], cwd=output, check=True) + archive.write_bytes(b"corrupt") + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + package.assemble(archive, checksums, root / "corrupt-output") + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/operations/debian.md b/docs/operations/debian.md new file mode 100644 index 00000000..bd89ab20 --- /dev/null +++ b/docs/operations/debian.md @@ -0,0 +1,137 @@ +# Debian packages + +The initial apt target is **Ubuntu 24.04 amd64**. Install the `debuglet` package +from a reviewed signed repository following [repository setup](apt-repository.md). +Use the independently supplied OpenPGP fingerprint and a dedicated `Signed-By` +keyring; an archive checksum alone does not establish trust. No official live +repository URL is assumed by these instructions. + +`debuglet` selects the commands `/usr/bin/dbl`, `/usr/bin/debuglet-executor` and +`/usr/bin/debuglet-dispatcher`. Its exact dependency is a separately named +`debuglet-payload-` package. That package preserves every +file of the existing full tar payload at +`/usr/lib/debuglet/debian//`, including its original +manifest, license, samples and all three binaries. Every release gets a separate +payload directory. Apt does not overwrite an older release during an upgrade. + +Installation creates a missing `debuglet` system account and group, with no login +shell. Existing accounts and groups are retained. Missing `/var/lib/debuglet` +and `/var/lib/debuglet/executors` directories are created with mode `0755`; the +new executors parent belongs to the `debuglet` account and its primary group. +Existing ownership and modes are preserved. No package script enrolls an +executor, creates a per-executor identity, starts or restarts a service, changes +trust or configuration, or migrates a database. Python 3 is required for the +read-only removal guard; it adds no background process. + +## Enrollment and activation + +Use `/usr/bin/dbl` explicitly if an older tar-installed command appears earlier +on `PATH`. Follow [executor onboarding](executor-onboarding.md) to register, +establish trust and join using the service account. Joining creates the named +executor's state directory under `/var/lib/debuglet/executors`. Install and +activate that enrolled executor explicitly with the documented service command. +Dispatcher and local demo setup remain explicit as described in +[services](services.md). + +## Updates and rollback + +An apt update stages a new full payload and switches the three command links. +It leaves running and stopped managed services on their previously recorded +absolute payload paths. Even `apt upgrade` does not restart, enroll, migrate or +replace a managed service. Inspect the selected payload with: + +```sh +dpkg-query -W -f='${Version}\n' debuglet +candidate=$(dirname "$(dirname "$(readlink -f /usr/bin/dbl)")") +printf '%s\n' "$candidate" +``` + +For an enrolled executor, follow the [release rollout](release-rollout.md): +drain it, create and verify an offline backup of its complete identity, keys, +trust, configuration, database, outputs and service records, then use the +verified replacement command while the unit is stopped and disabled: + +```sh +sudo /usr/bin/dbl service install --role executor --name worker \ + --enrolled-state /var/lib/debuglet/executors/worker \ + --replace-package "$candidate" --start=false --enable=false +``` + +Use the recorded instance name instead of `worker`. Replacement preserves its +service account, state paths and identity. Check compatibility and explicitly +resume using the rollout procedure. Rollback selects a retained older payload +with the same replacement command and is subject to database compatibility; +an apt downgrade alone does not roll back a service or a database. Existing +dispatcher replacement limits are unchanged; see the rollout documentation. + +Release `v1.2.3` has Debian version `1.2.3-1`; prerelease `v1.2.3-rc.1` becomes +`1.2.3~rc.1-1`, which sorts before the final release. Development versions such +as `v0.0.0-dev.abcdef012345` become `0.0.0~dev.abcdef012345-1`. Commit hashes do +not sort chronologically. Development candidates must be selected by their +exact Debian version; use apt's explicit downgrade option only when intentionally +selecting a version that sorts lower. + +## Removal and repair + +Removing or purging `debuglet` removes its command links and documentation. +It does not stop a service or delete state, identities, accounts or outputs. +The payload package's removal guard refuses removal whenever a retained +managed-service record references it, a command link in `/usr/bin` or +`/usr/local/bin` selects it, or a process is executing it. Unknown process state +and malformed or unreadable service records also refuse removal. No service +manager call is required for these checks. Run package maintenance separately +from enrollment, service replacement and foreground role startup so a new +reference cannot be created during dpkg's removal operation. + +Apt may mark old payload dependencies as eligible for `autoremove`. The guard +still protects referenced payloads; refusal makes that cleanup command fail +with actionable guidance and leaves the protected payload in place. Use +`apt-mark manual ` to keep a known recovery version out of +automatic cleanup. When a payload is no longer selected, executing or referenced, +remove its exact package with apt. An uninstalled service's retained record +still protects its payload. `dbl service uninstall --purge` also deletes that +role's data; do not use it merely to free software disk space. Preserve the +record and payload when recovery is still needed. + +Same-version `apt-get --reinstall install =` +restores the exact signed package bytes and does not restart services or alter +state. It also supports retry after interrupted unpack, followed by +`dpkg --configure -a`. A directory replaced by a symlink is refused; investigate +and restore the real directory before retrying. Different payload content uses +a different package name and directory; never republish changed content under +an existing package identity/version. A forced dpkg removal that skips guards, +manual file deletion or custom unmanaged unit references is outside this +managed lifecycle. + +Tar installations remain supported under their existing prefix, normally +`/usr/local`. Their version paths differ from Debian's content-addressed paths. +`dbl service prune` accepts only a valid version at `PREFIX/lib/debuglet/VERSION` +(or a known component subdirectory), so it cannot target a Debian payload. +The tar installer also refuses to replace the differently shaped Debian command +link at `/usr/bin/dbl`. Use apt for Debian-owned files and tar pruning for tar +versions. + +## Build and inspect + +After the normal clean-source full tar build, on Linux amd64 with Python 3 and +`dpkg-deb` installed: + +```sh +scripts/package-deb.sh --package-dir .cache/ci/packages +``` + +The default output is `.cache/ci/packages/debian`. `--out DIR` chooses another +empty directory. Explicit inputs use `--archive FILE --checksums FILE --out DIR` +with the matching generated `install.sh` beside the archive. The existing +installer verifies archive hashes, exact payload members, modes and bytes before +assembly. The output contains `debuglet-VERSION-linux-amd64.deb`, +`debuglet-payload-VERSION-linux-amd64.deb`, `deb-packages.json` and +`SHA256SUMS-debian`. Both packages must be published and indexed together. +The JSON binds the source revision, archive, manifest digest, Debian version, +payload identity and package checksums. Release verification additionally checks +the `.deb` file contents and maintainer scripts against the reviewed source. + +Run focused assembly and guard tests with +`python3 -m unittest discover -s deploy/debian -p 'test_*.py'`. +Use the Debian acceptance job for actual apt installation, upgrade, repair, +retained-service behavior and signed repository trust tests. diff --git a/scripts/package-deb.sh b/scripts/package-deb.sh new file mode 100755 index 00000000..8e2a6414 --- /dev/null +++ b/scripts/package-deb.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +exec python3 deploy/debian/package.py "$@" From 4d948fbf3982d4f3878ad5184dab4a8ce6d4da58 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 13:57:30 +0200 Subject: [PATCH 03/10] Normalize Debian package directories independently of builder umask --- deploy/debian/package.py | 3 +++ deploy/debian/test_package.py | 18 +++++++++++++++++- docs/operations/debian.md | 5 +++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/deploy/debian/package.py b/deploy/debian/package.py index ac5ed274..06976f84 100755 --- a/deploy/debian/package.py +++ b/deploy/debian/package.py @@ -114,7 +114,10 @@ def assemble(archive, checksums, output): write(documentation / "debian.md", (REPO / "docs/operations/debian.md").read_text()) # Stable timestamps/root ownership keep builds independent of the # invoking user. gzip is supported by Ubuntu's dpkg without extras. + tree.chmod(0o755) for path in tree.rglob("*"): + if path.is_dir() and not path.is_symlink(): + path.chmod(0o755) os.utime(path, (0, 0), follow_symlinks=False) filename = "debuglet" if name == "debuglet" else "debuglet-payload" package_file = output / f"{filename}-{upstream}-linux-amd64.deb" diff --git a/deploy/debian/test_package.py b/deploy/debian/test_package.py index e69bbf98..c38c5b66 100644 --- a/deploy/debian/test_package.py +++ b/deploy/debian/test_package.py @@ -129,8 +129,18 @@ def test_full_payload_manifest_and_modes_are_unchanged(self): checksums = root / "SHA256SUMS" checksums.write_text("".join(f"{package.digest(path)} {path.name}\n" for path in (archive, root / "install.sh"))) output = root / "debs" - result = package.assemble(archive, checksums, output) + old_umask = os.umask(0o077) + try: + result = package.assemble(archive, checksums, output) + finally: + os.umask(old_umask) payload = output / f"debuglet-payload-{upstream}-linux-amd64.deb" + with tarfile.open(fileobj=io.BytesIO(subprocess.check_output( + ["dpkg-deb", "--fsys-tarfile", str(payload)]))) as package_tar: + for member in package_tar: + self.assertEqual((member.uid, member.gid), (0, 0)) + if member.isdir(): + self.assertEqual(member.mode, 0o755, member.name) extracted = root / "extracted" subprocess.run(["dpkg-deb", "--extract", str(payload), str(extracted)], check=True) installed = extracted / result["payload_root"].lstrip("/") @@ -141,6 +151,12 @@ def test_full_payload_manifest_and_modes_are_unchanged(self): # A Debian hash root has no such valid version path component. self.assertIsNone(package.VERSION.fullmatch(installed.name)) selector = output / f"debuglet-{upstream}-linux-amd64.deb" + with tarfile.open(fileobj=io.BytesIO(subprocess.check_output( + ["dpkg-deb", "--fsys-tarfile", str(selector)]))) as package_tar: + for member in package_tar: + self.assertEqual((member.uid, member.gid), (0, 0)) + if member.isdir(): + self.assertEqual(member.mode, 0o755, member.name) subprocess.run(["dpkg-deb", "--extract", str(selector), str(root / "selector")], check=True) self.assertEqual(os.readlink(root / "selector/usr/bin/dbl"), "../lib/debuglet/debian/" + installed.name + "/bin/dbl") diff --git a/docs/operations/debian.md b/docs/operations/debian.md index bd89ab20..ae439285 100644 --- a/docs/operations/debian.md +++ b/docs/operations/debian.md @@ -95,8 +95,9 @@ record and payload when recovery is still needed. Same-version `apt-get --reinstall install =` restores the exact signed package bytes and does not restart services or alter -state. It also supports retry after interrupted unpack, followed by -`dpkg --configure -a`. A directory replaced by a symlink is refused; investigate +state. After interrupted unpack, run `sudo dpkg --configure -a` first, then +`sudo apt-get --fix-broken --reinstall install debuglet` to +finish configuration and restore missing files. A directory replaced by a symlink is refused; investigate and restore the real directory before retrying. Different payload content uses a different package name and directory; never republish changed content under an existing package identity/version. A forced dpkg removal that skips guards, From 5ee1cc2c2f6edf64ccaa2b708a556d5b5b6af6ce Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 13:55:33 +0200 Subject: [PATCH 04/10] Add signed apt repository publication and retention --- .github/workflows/apt-publish.yml | 174 +++++++++++++++ docs/operations/apt-repository.md | 199 +++++++++++++++++ tools/apt_repository.py | 350 ++++++++++++++++++++++++++++++ tools/test_apt_repository.py | 186 ++++++++++++++++ 4 files changed, 909 insertions(+) create mode 100644 .github/workflows/apt-publish.yml create mode 100644 docs/operations/apt-repository.md create mode 100644 tools/apt_repository.py create mode 100644 tools/test_apt_repository.py diff --git a/.github/workflows/apt-publish.yml b/.github/workflows/apt-publish.yml new file mode 100644 index 00000000..49f0b1d7 --- /dev/null +++ b/.github/workflows/apt-publish.yml @@ -0,0 +1,174 @@ +name: Publish signed apt repository + +on: + workflow_dispatch: + inputs: + bootstrap: + description: First publication only (requires no prior Pages deployments) + required: true + default: false + type: boolean + +permissions: + contents: read + actions: read + +concurrency: + group: apt-pages-publication + cancel-in-progress: false + +jobs: + prepare: + if: github.repository == 'netsec-ethz/debuglet' && github.ref_type == 'tag' && github.ref_protected + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + actions: read + pages: read + deployments: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Check exact approved release inputs and publication settings + env: + GH_TOKEN: ${{ github.token }} + APT_PUBLIC_KEY: ${{ vars.APT_PUBLIC_KEY }} + APT_KEY_FINGERPRINT: ${{ vars.APT_KEY_FINGERPRINT }} + APT_REPOSITORY_URL: ${{ vars.APT_REPOSITORY_URL }} + BOOTSTRAP: ${{ inputs.bootstrap }} + shell: bash + run: | + set -euo pipefail + [[ -n "$APT_PUBLIC_KEY" && -n "$APT_KEY_FINGERPRINT" && -n "$APT_REPOSITORY_URL" ]] + mkdir .apt-input + printf '%s\n' "$APT_PUBLIC_KEY" > .apt-input/key.asc + gh api "repos/$GITHUB_REPOSITORY/environments/release-signing" > .apt-input/environment.json + python3 - <<'PY' + import json, sys + sys.path.insert(0, 'tools') + from release import check_environment + check_environment(json.load(open('.apt-input/environment.json'))) + PY + gh api "repos/$GITHUB_REPOSITORY/pages" > .apt-input/pages.json + python3 - <<'PY' + import json, os + pages = json.load(open('.apt-input/pages.json')) + assert pages.get('build_type') == 'workflow', 'Pages must use GitHub Actions' + assert pages.get('https_enforced') is True, 'Pages must enforce HTTPS' + assert pages.get('html_url', '').rstrip('/') == os.environ['APT_REPOSITORY_URL'].rstrip('/'), 'configured repository URL differs from Pages' + PY + if [[ "$BOOTSTRAP" == true ]]; then + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/deployments?environment=github-pages&per_page=100" \ + --jq '[.[][]]' > .apt-input/deployments.json + python3 -c 'import json; assert json.load(open(".apt-input/deployments.json")) == [], "bootstrap refused: prior Pages deployments exist"' + fi + - name: Download the published complete signed release + env: + GH_TOKEN: ${{ github.token }} + run: python3 tools/apt_repository.py download-release --version "$GITHUB_REF_NAME" --out .apt-input/release + - name: Verify signed release and retain every previously published package + env: + APT_KEY_FINGERPRINT: ${{ vars.APT_KEY_FINGERPRINT }} + APT_REPOSITORY_URL: ${{ vars.APT_REPOSITORY_URL }} + BOOTSTRAP: ${{ inputs.bootstrap }} + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install --no-install-recommends -y apt-utils gnupg + fetch_options=() + previous_options=() + if [[ "$BOOTSTRAP" == true ]]; then + fetch_options+=(--bootstrap) + else + previous_options+=(--previous "$PWD/.apt-input/previous") + fi + python3 tools/apt_repository.py fetch --url "$APT_REPOSITORY_URL" --out .apt-input/previous \ + --key .apt-input/key.asc --fingerprint "$APT_KEY_FINGERPRINT" "${fetch_options[@]}" + python3 tools/apt_repository.py prepare --release-directory .apt-input/release \ + --version "$GITHUB_REF_NAME" --source "$GITHUB_SHA" \ + --trust configs/release-allowed-signers --signer releases@debuglet.netsec.ethz.ch \ + --key .apt-input/key.asc --fingerprint "$APT_KEY_FINGERPRINT" \ + --out .apt-site "${previous_options[@]}" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: prepared-apt-${{ github.run_id }}-${{ github.run_attempt }} + path: .apt-site/ + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + + sign: + needs: prepare + if: github.repository == 'netsec-ethz/debuglet' && github.ref_type == 'tag' && github.ref_protected + runs-on: ubuntu-24.04 + timeout-minutes: 10 + environment: release-signing + permissions: + actions: read + steps: + # No checkout, package extraction, installer, or artifact-supplied code runs here. + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: prepared-apt-${{ github.run_id }}-${{ github.run_attempt }} + path: site + - name: Sign apt Release with independently provisioned OpenPGP trust + env: + APT_SIGNING_KEY: ${{ secrets.APT_SIGNING_KEY }} + APT_PUBLIC_KEY: ${{ vars.APT_PUBLIC_KEY }} + APT_KEY_FINGERPRINT: ${{ vars.APT_KEY_FINGERPRINT }} + shell: bash + run: | + set +x + set -euo pipefail + umask 077 + [[ "$APT_KEY_FINGERPRINT" =~ ^[0-9A-F]{40}$|^[0-9A-F]{64}$ ]] + [[ -n "$APT_SIGNING_KEY" && -n "$APT_PUBLIC_KEY" ]] + export GNUPGHOME + GNUPGHOME=$(mktemp -d "$RUNNER_TEMP/apt-key.XXXXXXXX") + trap 'gpgconf --kill gpg-agent || true; rm -rf -- "$GNUPGHOME"' EXIT + printf '%s\n' "$APT_SIGNING_KEY" | gpg --batch --import + unset APT_SIGNING_KEY + printf '%s\n' "$APT_PUBLIC_KEY" > "$GNUPGHOME/public.asc" + gpg --batch --dearmor --output "$GNUPGHOME/trusted.gpg" "$GNUPGHOME/public.asc" + primary=$(gpg --batch --with-colons --show-keys --fingerprint "$GNUPGHOME/public.asc" | awk -F: '$1=="pub" {want=1; next} $1=="fpr" && want {print $10; want=0}') + [[ "$primary" == "$APT_KEY_FINGERPRINT" ]] + cmp "$GNUPGHOME/public.asc" site/debuglet-archive-keyring.asc + gpg --batch --yes --local-user "$APT_KEY_FINGERPRINT" --digest-algo SHA256 \ + --clearsign --output site/dists/stable/InRelease site/dists/stable/Release + gpg --batch --yes --local-user "$APT_KEY_FINGERPRINT" --digest-algo SHA256 --armor \ + --detach-sign --output site/dists/stable/Release.gpg site/dists/stable/Release + gpgv --keyring "$GNUPGHOME/trusted.gpg" --output "$GNUPGHOME/verified-release" site/dists/stable/InRelease + cmp "$GNUPGHOME/verified-release" site/dists/stable/Release + gpgv --keyring "$GNUPGHOME/trusted.gpg" site/dists/stable/Release.gpg site/dists/stable/Release + gpgconf --kill gpg-agent || true + rm -rf -- "$GNUPGHOME" + trap - EXIT + - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 + with: + path: site + retention-days: 7 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: signed-apt-${{ github.ref_name }}-${{ github.sha }}-${{ github.run_attempt }} + path: site/ + include-hidden-files: true + if-no-files-found: error + retention-days: 90 + + deploy: + needs: sign + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + id: deployment diff --git a/docs/operations/apt-repository.md b/docs/operations/apt-repository.md new file mode 100644 index 00000000..7f7c4f1b --- /dev/null +++ b/docs/operations/apt-repository.md @@ -0,0 +1,199 @@ +# Signed apt repository + +The apt channel serves Ubuntu 24.04 amd64. Installing `debuglet` stages the CLI +and its exact full payload; enrollment, service activation and migration remain +explicit operations. See [Debian installation](debian.md) for package and service +lifecycle behavior. The existing [tar installation](releases.md) remains supported. + +The publishing workflow requires administrator setup and an approved protected +release before the channel can be used. This document does not establish a live +repository URL or production OpenPGP fingerprint. Configure website onboarding +only after the actual HTTPS download and apt installation have been verified. + +## Trust and layout + +APT authenticates `dists/stable/InRelease`, which contains the signed `Release` +metadata. Its SHA256 entries bind `main/binary-amd64/Packages` and `Packages.gz`; +each package entry binds the exact `.deb` bytes and size. A detached +`Release.gpg` is also published. `apt-ftparchive` generates the indexes and Release; +GnuPG signs them. Packages are never executed during repository preparation or +signing. + +The repository has suite `stable`, component `main`, architecture `amd64`, and +publishes its ASCII public key at `debuglet-archive-keyring.asc`. The key must be +confirmed against a fingerprint obtained through an independent authenticated +channel. Downloading a key beside a package does not authenticate the publisher. +The existing OpenSSH release key authenticates the input release bundle; apt +uses a separate OpenPGP key and does not understand that OpenSSH signature. + +Every publication retains all previously indexed selector and payload packages +under `pool/main/d/debuglet/`. Apt pool filenames are derived from Debian control +fields; their bytes remain identical to the signed release subjects. Existing +package bytes cannot be replaced, and a selector older than any retained selector +cannot be published. Apt can download retained exact versions for a deliberate +rollback; this does not establish state downgrade compatibility. Follow the +backup, drain and restore procedures before switching a service's release. + +Content-addressed indexes support `Acquire-By-Hash`, including the immediately +previous generation to cover clients updating during an atomic deployment. +Signed metadata expires after 90 days. Before expiry, rerun publication of the +current approved release to refresh its Date and Valid-Until while retaining all +packages. Do not disable apt expiry or signature checks. Repository preparation +may verify an expired previous snapshot solely to retain its packages for a new +signed publication; this is not permission for clients to install from expired +metadata. + +## Administrator setup + +1. Review and merge the complete package and publication changes. Protect the + intended semantic-version tags and have an authorized tag creator create the + chosen release tag at the exact reviewed source. A branch build is not a + release input. Run the full required CI for that tag, then **Prepare signed + release** with that completed CI run ID. The existing independent + `release-signing` reviewer must approve its signing job. Audit and publish the + complete signed bundle as assets on the GitHub release for that same tag; + publication of only the installation subset is insufficient for apt. +2. Provision a dedicated OpenPGP signing key through the team's key-management + process. Retain a protected backup and revocation material. Put its exported + private signing key in the `APT_SIGNING_KEY` secret of `release-signing` only; + do not use a repository or organization secret as a fallback. The unattended + GnuPG invocation expects an export usable without an interactive passphrase. + Keep the existing `RELEASE_SIGNING_KEY` unchanged: it is an OpenSSH key for a + different format. +3. Keep `release-signing` protected by required independent reviewers with + prevent-self-review enabled, and restrict it to the approved version tags. + Create or review the `github-pages` environment with an independent reviewer + and deployment rules allowing only those release tags. Do not broaden tag + creation or environment rules to make a maintainer's blocked run pass. +4. Enable GitHub Pages for this repository with **GitHub Actions** as the source, + configure the desired domain if applicable, and enforce HTTPS. This workflow + owns the entire Pages site; do not share that destination with another Pages + publisher. Record its actual `html_url`, including any repository path. +5. Set repository variables `APT_REPOSITORY_URL` to that exact HTTPS Pages URL, + `APT_PUBLIC_KEY` to the single ASCII-armored public key, and + `APT_KEY_FINGERPRINT` to its full uppercase primary-key fingerprint without + spaces. The workflow checks the URL against Pages and checks the key against + the fingerprint. Distribute the fingerprint independently to operators and + the website configuration reviewer. No production key is generated by CI. + +The workflow also checks that `release-signing` still has independent review +before preparing any repository. GitHub enforces the environment approvals and +tag/deployment rules. Its signing job checks out no source, has no Contents write +permission, and runs only GnuPG against prepared metadata. Signing-key material +is removed before uploading the signed site. The deploy job gets Pages write +and OIDC permissions, and receives no signing key. + +## Publish and verify + +Dispatch **Publish signed apt repository** on the same protected version tag. +The publisher downloads the complete assets from that tag's published, non-draft +GitHub release, then audits every SSH-signed subject and CI gate using +`tools/release.py`, pinning the expected source to the workflow's protected tag. +The input must contain both signed Debian packages and their assembly inventory; +legacy tar-only releases cannot enter the apt channel. No branch build or +expiring Actions artifact is accepted as a substitute for the published release. +Keep that complete release available so metadata refresh remains possible after +Actions artifacts expire. + +For the first publication only, set `bootstrap=true`. This requires no prior +`github-pages` deployment records and an HTTP 404 for the repository's InRelease. +A timeout, bad signature or failed download never becomes an empty repository. +A failed first deployment requires operator inspection before any bootstrap +retry; do not delete a successful deployment record to bypass retention. + +For subsequent publications leave bootstrap false. The workflow downloads the +existing InRelease and verifies it against the configured OpenPGP key, then +verifies the hashes of both indexes and every retained package. Missing, changed +or untrusted content stops publication. It then indexes the old and new packages, +requests the signing approval, and deploys the signed site after the Pages +environment's approval. Publication runs are serialized. + +The workflow saves a signed site artifact for 90 days, but Actions retention is +not a durable archive. Keep an independent backup of each approved signed site +and the complete signed release bundles. GitHub Pages capacity is finite; move +the channel to suitable static hosting before retention exceeds its capacity. +Do not silently prune payloads still needed by installed services or overwrite +old package versions to save space. + +After deployment, fetch and verify the real published bytes from a trusted +checkout with independently supplied key material: + +```sh +python3 tools/apt_repository.py fetch \ + --url "$DEBUGLET_APT_URL" --out /tmp/debuglet-apt-downloaded \ + --key /path/to/independently-provisioned-key.asc \ + --fingerprint "$DEBUGLET_APT_FINGERPRINT" +``` + +Use a fresh output directory. This verifies the actual remote metadata and all +packages without executing them. Complete a fresh Ubuntu 24.04 installation, +explicit enrollment, and a retained-service upgrade using those downloaded +bytes before recording the channel as tested. A local test repository or a +successful upload step does not establish public-channel success. + +## Configure a client + +Obtain the real URL, complete uppercase fingerprint and published Debian version +from the approved release information. The following variables are placeholders, +not production trust. Run these commands as the intended host's administrator: + +```sh +export DEBUGLET_APT_URL=https://YOUR-APPROVED-HOST/YOUR-REPOSITORY-PATH +export DEBUGLET_APT_FINGERPRINT=YOUR_INDEPENDENTLY_CONFIRMED_FULL_FINGERPRINT +export DEBUGLET_DEBIAN_VERSION=1.2.3-1 + +curl --fail --silent --show-error --proto '=https' --proto-redir '=https' \ + --location "$DEBUGLET_APT_URL/debuglet-archive-keyring.asc" -o /tmp/debuglet-key.asc +actual=$(gpg --batch --show-keys --with-colons --fingerprint /tmp/debuglet-key.asc | + awk -F: '$1=="pub" {want=1; next} $1=="fpr" && want {print $10; want=0}') +test "$actual" = "$DEBUGLET_APT_FINGERPRINT" || exit 1 +gpg --batch --yes --dearmor --output /tmp/debuglet-key.gpg /tmp/debuglet-key.asc +sudo install -d -m 0755 /etc/apt/keyrings +sudo install -m 0644 /tmp/debuglet-key.gpg /etc/apt/keyrings/debuglet.gpg +printf 'Types: deb\nURIs: %s\nSuites: stable\nComponents: main\nArchitectures: amd64\nSigned-By: /etc/apt/keyrings/debuglet.gpg\n' \ + "$DEBUGLET_APT_URL" | sudo tee /etc/apt/sources.list.d/debuglet.sources >/dev/null +sudo apt-get update +sudo apt-get install "debuglet=$DEBUGLET_DEBIAN_VERSION" +``` + +`Signed-By` limits this trust to the Debuglet source. Do not use `apt-key`, +`trusted=yes`, `--allow-unauthenticated`, or an insecure fallback after a failed +verification. The example upstream version `v1.2.3` maps to Debian `1.2.3-1`; +`v1.2.3-rc.1` maps to `1.2.3~rc.1-1`. + +A later `apt-get update` followed by `apt-get install debuglet=EXACT_VERSION` +stages the selected release. It does not itself drain, back up, migrate, enroll +or restart a managed Debuglet service. Refer to the package lifecycle procedure +before activating that release. + +## Local rehearsal and key rotation + +The unsigned builder is available for isolated tests: + +```sh +python3 tools/apt_repository.py build --packages /path/to/debian-artifacts \ + --out /tmp/apt-test-site +``` + +Sign `dists/stable/Release` using a disposable test key to create `InRelease` +and `Release.gpg`, then run `verify --directory /tmp/apt-test-site --key TEST.asc +--fingerprint TEST_FINGERPRINT`. `build --previous VERIFIED_DIRECTORY` retains +all packages from a previously verified snapshot. This local builder performs +no release authentication; production must use `prepare`, as the workflow does. +HTTP is refused except with explicit `--allow-local-http` for loopback fixtures. +Never provision a rehearsal key as production trust. + +Changing the configured public key immediately prevents the current workflow +from trusting an old snapshot signed solely by the previous key. Plan rotation +as a reviewed publication change: independently distribute the new key, verify +and retain the entire old repository with its old trust, and authorize a new +signature over those retained bytes with the new key. Update client keyrings +before ending the overlap. The single-key workflow intentionally does not guess +rotation trust or bypass a signature failure. In a compromise, revoke the old +key and use an independently authenticated retained snapshot; do not accept a +snapshot solely because the compromised key signed it. + +APT's [repository authentication](https://manpages.debian.org/stable/apt/apt-secure.8.en.html) +and [apt-ftparchive](https://manpages.debian.org/stable/apt-utils/apt-ftparchive.1.en.html) +document the formats. GitHub documents the required +[Pages workflow permissions and environments](https://docs.github.com/en/pages/getting-started-with-github-pages/using-custom-workflows-with-github-pages). diff --git a/tools/apt_repository.py b/tools/apt_repository.py new file mode 100644 index 00000000..bd79345c --- /dev/null +++ b/tools/apt_repository.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Build and verify the small Debuglet apt repository with standard Debian tools.""" +import argparse +import gzip +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime +import io +import hashlib +import json +import re +import shutil +import subprocess +import tempfile +from pathlib import Path +from urllib.error import HTTPError +from urllib.parse import urlsplit +from urllib.request import urlopen + +SUITE = Path('dists/stable') +INDEX = Path('main/binary-amd64/Packages') +POOL = Path('pool/main/d/debuglet') +MAX_INDEX = 16 * 1024 * 1024 +MAX_PACKAGE = 512 * 1024 * 1024 +PACKAGE = re.compile(r'debuglet(?:-payload-[0-9a-f]{24})?\Z') +FILENAME = re.compile(r'debuglet(?:-payload-[0-9a-f]{24})?_[0-9][0-9A-Za-z.+~:-]*_amd64\.deb\Z') + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def command(args, **kwargs): + return subprocess.check_output(args, timeout=180, **kwargs) + + +def regular(path, maximum=MAX_PACKAGE): + require(path.is_file() and not path.is_symlink() and path.stat().st_size <= maximum, + 'missing, linked or oversized file: ' + str(path)) + return path.read_bytes() + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def fields(data): + """Read the Deb822 records emitted by apt-ftparchive, rejecting ambiguity.""" + result = [] + for paragraph in data.decode('utf-8').strip().split('\n\n'): + record, key = {}, None + for line in paragraph.splitlines(): + if line.startswith((' ', '\t')): + require(key is not None, 'orphan metadata continuation') + record[key] += '\n' + line.strip() + else: + require(':' in line, 'invalid apt metadata field') + key, value = line.split(':', 1) + require(key not in record and re.fullmatch(r'[A-Za-z0-9-]+', key), + 'duplicate or invalid apt metadata field') + record[key] = value.strip() + require(record, 'empty apt metadata') + result.append(record) + return result + + +def release_indexes(data): + records = fields(data) + require(len(records) == 1, 'expected one Release record') + release = records[0] + for key, value in {'Origin': 'Debuglet', 'Label': 'Debuglet', 'Suite': 'stable', + 'Codename': 'stable', 'Architectures': 'amd64', 'Components': 'main'}.items(): + require(release.get(key) == value, 'unexpected Release ' + key) + result = {} + for line in release.get('SHA256', '').splitlines(): + if not line: + continue + parts = line.split() + require(len(parts) == 3 and re.fullmatch(r'[0-9a-f]{64}', parts[0]) + and parts[1].isdigit(), 'invalid Release SHA256 entry') + hashed, size, name = parts + require(name not in result and name in (str(INDEX), str(INDEX) + '.gz') + and 0 < int(size) <= MAX_INDEX, 'unexpected, duplicate or oversized apt index') + result[name] = (hashed, int(size)) + require(set(result) == {str(INDEX), str(INDEX) + '.gz'}, 'incomplete apt indexes') + return result + + +def package_entries(data): + result, identities = {}, set() + for record in fields(data): + name, version, architecture = (record.get(key, '') for key in ('Package', 'Version', 'Architecture')) + filename = record.get('Filename', '') + require(PACKAGE.fullmatch(name) and architecture == 'amd64' + and re.fullmatch(r'[0-9][0-9A-Za-z.+~:-]*', version), 'unexpected package identity') + require(filename == str(POOL / f'{name}_{version}_amd64.deb') + and FILENAME.fullmatch(Path(filename).name), 'unexpected package filename') + require(filename not in result and (name, version, architecture) not in identities, + 'duplicate package identity') + require(re.fullmatch(r'[0-9a-f]{64}', record.get('SHA256', '')) + and record.get('Size', '').isdigit() + and 0 < int(record['Size']) <= MAX_PACKAGE, 'invalid package digest or size') + result[filename] = (record['SHA256'], int(record['Size'])) + identities.add((name, version, architecture)) + require(any(record['Package'] == 'debuglet' for record in fields(data)), 'missing debuglet selector') + return result + + +def matches(path, expected, maximum): + data = regular(path, maximum) + require((digest(data), len(data)) == expected, 'SHA256/size mismatch: ' + str(path)) + return data + + +def keyring(key, fingerprint, directory): + fingerprint = fingerprint.upper() + require(re.fullmatch(r'[0-9A-F]{40}|[0-9A-F]{64}', fingerprint), 'invalid OpenPGP fingerprint') + regular(key, 1024 * 1024) + home = directory / 'gnupg' + home.mkdir(mode=0o700) + output = command(['gpg', '--batch', '--no-options', '--homedir', str(home), '--with-colons', + '--show-keys', '--fingerprint', str(key)]).decode() + primary, pending = [], False + for line in output.splitlines(): + parts = line.split(':') + if parts[0] == 'pub': + pending = True + elif parts[0] == 'fpr' and pending: + primary.append(parts[9]) + pending = False + require(primary == [fingerprint], 'public key does not match the independently provisioned fingerprint') + ring = directory / 'trusted.gpg' + command(['gpg', '--batch', '--no-options', '--homedir', str(home), '--dearmor', + '--output', str(ring), str(key)]) + return ring + + +def verified_release(directory, key, fingerprint): + regular(directory / SUITE / 'InRelease', MAX_INDEX) + with tempfile.TemporaryDirectory(prefix='debuglet-apt-verify-') as temporary: + temporary = Path(temporary) + ring = keyring(key, fingerprint, temporary) + output = temporary / 'Release' + command(['gpgv', '--homedir', str(temporary / 'gnupg'), '--keyring', str(ring), + '--output', str(output), str(directory / SUITE / 'InRelease')]) + data = regular(output, MAX_INDEX) + release_indexes(data) + return data + + +def verify(directory, key, fingerprint): + release = verified_release(directory, key, fingerprint) + require(regular(directory / SUITE / 'Release', MAX_INDEX) == release, + 'Release differs from verified InRelease') + for name, expected in release_indexes(release).items(): + matches(directory / SUITE / name, expected, MAX_INDEX) + packages = regular(directory / SUITE / INDEX, MAX_INDEX) + with gzip.GzipFile(fileobj=io.BytesIO(regular(directory / SUITE / (str(INDEX) + '.gz'), MAX_INDEX))) as compressed: + require(compressed.read(MAX_INDEX + 1) == packages, 'compressed index differs') + entries = package_entries(packages) + for name, expected in entries.items(): + matches(directory / name, expected, MAX_PACKAGE) + actual = {p.relative_to(directory).as_posix() for p in (directory / 'pool').rglob('*') if p.is_file()} + require(actual == set(entries), 'unsigned extra pool files') + return {'packages': len(entries), 'release_sha256': digest(release), 'verified': True} + + +def fetch(url, directory, key, fingerprint, bootstrap=False, allow_local_http=False): + parsed = urlsplit(url) + allowed = parsed.scheme == 'https' or (allow_local_http and parsed.scheme == 'http' + and parsed.hostname in ('localhost', '127.0.0.1', '::1')) + require(allowed and parsed.netloc and not parsed.username and not parsed.password + and not parsed.query and not parsed.fragment, 'repository URL must be HTTPS without credentials') + directory.mkdir() + + def download(name, limit): + target = directory / name + target.parent.mkdir(parents=True, exist_ok=True) + with urlopen(url.rstrip('/') + '/' + name, timeout=60) as response: + final = urlsplit(response.url) + require(final.scheme == 'https' or (allow_local_http and final.scheme == 'http' + and final.hostname in ('localhost', '127.0.0.1', '::1')), 'unsafe repository redirect') + data = response.read(limit + 1) + require(len(data) <= limit, 'repository file exceeds size limit') + target.write_bytes(data) + + try: + download(str(SUITE / 'InRelease'), MAX_INDEX) + except HTTPError as error: + if bootstrap and error.code == 404: + return {'bootstrap': True, 'packages': 0} + raise + require(not bootstrap, 'bootstrap refused: a signed repository already exists') + release = verified_release(directory, key, fingerprint) + (directory / SUITE / 'Release').write_bytes(release) + for name, expected in release_indexes(release).items(): + download(str(SUITE / name), MAX_INDEX) + matches(directory / SUITE / name, expected, MAX_INDEX) + for name, expected in package_entries(regular(directory / SUITE / INDEX, MAX_INDEX)).items(): + download(name, MAX_PACKAGE) + matches(directory / name, expected, MAX_PACKAGE) + return verify(directory, key, fingerprint) + + +def build(packages, directory, previous=None): + """Create unsigned metadata; production callers first authenticate all inputs.""" + require(not directory.exists(), 'output already exists') + sources = sorted(packages.glob('*.deb')) + require(sources, 'no Debian packages') + directory.mkdir() + pool = directory / POOL + pool.mkdir(parents=True) + if previous is not None: + # The production prepare command verifies previous before reaching here. + entries = package_entries(regular(previous / SUITE / INDEX, MAX_INDEX)) + for name, expected in entries.items(): + matches(previous / name, expected, MAX_PACKAGE) + shutil.copyfile(previous / name, directory / name) + for source in sources: + data = regular(source) + output = command(['dpkg-deb', '--field', str(source), 'Package', 'Version', 'Architecture']) + control = fields(output)[0] + require(PACKAGE.fullmatch(control.get('Package', '')) and control.get('Architecture') == 'amd64' + and re.fullmatch(r'[0-9][0-9A-Za-z.+~:-]*', control.get('Version', '')), + 'unexpected Debian control identity') + if previous is not None and control['Package'] == 'debuglet': + for retained in fields(regular(previous / SUITE / INDEX, MAX_INDEX)): + if retained['Package'] == 'debuglet': + comparison = subprocess.run(['dpkg', '--compare-versions', control['Version'], + 'ge', retained['Version']], check=False, timeout=10) + require(comparison.returncode == 0, 'refusing to publish an older selector version') + destination = pool / '{Package}_{Version}_{Architecture}.deb'.format(**control) + require(not destination.exists() or regular(destination) == data, 'refusing to replace a retained package') + destination.write_bytes(data) + index = directory / SUITE / INDEX + index.parent.mkdir(parents=True) + data = command(['apt-ftparchive', 'packages', 'pool'], cwd=directory) + package_entries(data) + index.write_bytes(data) + (index.parent / 'Packages.gz').write_bytes(gzip.compress(data, mtime=0)) + # Keep the previous generation's content-addressed indexes for clients that + # fetched its InRelease immediately before an atomic Pages deployment. + by_hash = index.parent / 'by-hash/SHA256' + by_hash.mkdir(parents=True) + for root in (previous, directory): + if root is not None: + for name in (INDEX, Path(str(INDEX) + '.gz')): + contents = regular(root / SUITE / name, MAX_INDEX) + (by_hash / digest(contents)).write_bytes(contents) + options = {'Origin': 'Debuglet', 'Label': 'Debuglet', 'Suite': 'stable', 'Codename': 'stable', + 'Architectures': 'amd64', 'Components': 'main', 'Acquire-By-Hash': 'yes', + 'Description': 'Debuglet for Ubuntu 24.04 amd64', + 'Valid-Until': format_datetime(datetime.now(timezone.utc) + timedelta(days=90), usegmt=True)} + arguments = ['apt-ftparchive'] + for key, value in options.items(): + arguments += ['-o', f'APT::FTPArchive::Release::{key}={value}'] + release = command(arguments + ['release', str(SUITE)], cwd=directory) + release_indexes(release) + (directory / SUITE / 'Release').write_bytes(release) + (directory / '.nojekyll').touch() + return {'packages': len(package_entries(data)), 'release_sha256': digest(release), 'signed': False} + + +def prepare(release_directory, version, source, trust, signer, directory, key, fingerprint, previous=None): + from release import verify_release + release = verify_release(release_directory, trust, signer, version, source) + if previous is not None: + verify(previous, key, fingerprint) + names = [name for name in release['files'] if name.endswith('.deb')] + require(set(names) == {f'debuglet-{version}-linux-amd64.deb', + f'debuglet-payload-{version}-linux-amd64.deb'}, + 'release must bind the selector and one immutable payload package') + with tempfile.TemporaryDirectory(prefix='debuglet-apt-input-') as temporary: + staging = Path(temporary) + for name in names: + shutil.copyfile(release_directory / name, staging / name) + result = build(staging, directory, previous) + with tempfile.TemporaryDirectory(prefix='debuglet-apt-key-') as temporary: + keyring(key, fingerprint, Path(temporary)) + shutil.copyfile(key, directory / 'debuglet-archive-keyring.asc') + return dict(result, version=version, source_sha=source) + + +def check_published_release(record, version): + from release import VERSION + require(VERSION.fullmatch(version), 'invalid release version') + require(record.get('tag_name') == version and record.get('draft') is False + and isinstance(record.get('published_at'), str) and record['published_at'], + 'release must be published for the exact protected version tag') + + +def download_published(version, directory): + from release import REPOSITORY, VERSION, github_json, download_release + require(VERSION.fullmatch(version), 'invalid release version') + record = github_json(f'repos/{REPOSITORY}/releases/tags/{version}') + check_published_release(record, version) + directory.mkdir() + download_release(REPOSITORY, record, directory, version) + return {'downloaded': True, 'version': version, 'release_id': record['id']} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + for action in ('fetch', 'verify', 'prepare'): + command_parser = commands.add_parser(action) + command_parser.add_argument('--key', type=Path, required=True) + command_parser.add_argument('--fingerprint', required=True) + if action == 'verify': + command_parser.add_argument('--directory', type=Path, required=True) + else: + command_parser.add_argument('--out', type=Path, required=True) + if action == 'fetch': + command_parser.add_argument('--url', required=True) + command_parser.add_argument('--bootstrap', action='store_true') + command_parser.add_argument('--allow-local-http', action='store_true') + if action == 'prepare': + command_parser.add_argument('--release-directory', type=Path, required=True) + command_parser.add_argument('--version', required=True) + command_parser.add_argument('--source', required=True) + command_parser.add_argument('--trust', type=Path, required=True) + command_parser.add_argument('--signer', required=True) + command_parser.add_argument('--previous', type=Path) + command_parser = commands.add_parser('build', help='local unsigned repository rehearsal') + command_parser.add_argument('--packages', type=Path, required=True) + command_parser.add_argument('--out', type=Path, required=True) + command_parser.add_argument('--previous', type=Path) + command_parser = commands.add_parser('download-release') + command_parser.add_argument('--version', required=True) + command_parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + try: + if args.command == 'build': + result = build(args.packages.resolve(), args.out.resolve(), args.previous) + elif args.command == 'verify': + result = verify(args.directory, args.key, args.fingerprint) + elif args.command == 'fetch': + result = fetch(args.url, args.out, args.key, args.fingerprint, args.bootstrap, args.allow_local_http) + elif args.command == 'prepare': + result = prepare(args.release_directory, args.version, args.source, args.trust, args.signer, + args.out.resolve(), args.key, args.fingerprint, args.previous) + else: + result = download_published(args.version, args.out) + print(json.dumps(result, sort_keys=True)) + except (ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError) as error: + parser.exit(1, f'apt repository: {error}\n') + + +if __name__ == '__main__': + main() diff --git a/tools/test_apt_repository.py b/tools/test_apt_repository.py new file mode 100644 index 00000000..e0e16ba8 --- /dev/null +++ b/tools/test_apt_repository.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Focused repository tests. Run on Linux with apt-utils, dpkg and gnupg.""" +import http.server +from pathlib import Path +import shutil +import subprocess +import tempfile +import threading +import unittest + +import apt_repository as apt + + +class InputTests(unittest.TestCase): + def test_published_release_requires_exact_tag(self): + record = {'tag_name': 'v1.2.3', 'draft': False, 'published_at': '2026-10-15T12:00:00Z'} + apt.check_published_release(record, 'v1.2.3') + for field, invalid in [('tag_name', 'v1.2.2'), ('draft', True), ('published_at', None)]: + with self.subTest(field=field), self.assertRaises(ValueError): + apt.check_published_release(dict(record, **{field: invalid}), 'v1.2.3') + with self.assertRaises(ValueError): + apt.check_published_release(record, '../main') + + def test_duplicate_metadata_and_path_escape_refused(self): + with self.assertRaises(ValueError): + apt.fields(b'Package: debuglet\nPackage: other\n') + valid = ('Package: debuglet\nVersion: 1.2.3-1\nArchitecture: amd64\n' + 'Filename: ../../outside.deb\nSize: 1\nSHA256: ' + 'a' * 64 + '\n').encode() + with self.assertRaises(ValueError): + apt.package_entries(valid) + + def test_workflow_signer_has_no_checkout_or_payload_execution(self): + workflow = (Path(__file__).resolve().parents[1] / '.github/workflows/apt-publish.yml').read_text() + sign = workflow.split('\n sign:\n')[1].split('\n deploy:\n')[0] + self.assertNotIn('actions/checkout', sign) + self.assertNotIn('tools/apt_repository.py', sign) + self.assertIn('environment: release-signing', sign) + self.assertIn('github.ref_protected', sign) + self.assertIn('cmp "$GNUPGHOME/verified-release"', sign) + self.assertNotIn('APT_SIGNING_KEY', workflow.split('\n sign:\n')[0]) + + +@unittest.skipUnless(all(shutil.which(name) for name in ('apt-ftparchive', 'dpkg-deb', 'gpg', 'gpgv')), + 'requires Linux apt-utils, dpkg and gnupg') +class RepositoryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temporary = tempfile.TemporaryDirectory(prefix='debuglet-apt-tests-') + cls.root = Path(cls.temporary.name) + cls.home = cls.root / 'gnupg' + cls.home.mkdir(mode=0o700) + cls.gpg = ['gpg', '--batch', '--no-options', '--homedir', str(cls.home)] + cls.run_command(cls.gpg + ['--pinentry-mode', 'loopback', '--passphrase', '', + '--quick-generate-key', 'Disposable apt test ', + 'ed25519', 'sign', '0']) + records = cls.run_command(cls.gpg + ['--with-colons', '--list-keys']).decode().splitlines() + cls.fingerprint = next(line.split(':')[9] for line in records if line.startswith('fpr:')) + cls.key = cls.root / 'key.asc' + cls.key.write_bytes(cls.run_command(cls.gpg + ['--armor', '--export', cls.fingerprint])) + cls.old_packages = cls.packages('1.0.0-1', 'a' * 24) + cls.old = cls.root / 'old' + apt.build(cls.old_packages, cls.old) + cls.sign(cls.old) + cls.new_packages = cls.packages('1.1.0-1', 'b' * 24) + + @classmethod + def tearDownClass(cls): + cls.run_command(['gpgconf', '--homedir', str(cls.home), '--kill', 'gpg-agent']) + cls.temporary.cleanup() + + @staticmethod + def run_command(args): + return subprocess.check_output(args, stderr=subprocess.PIPE, timeout=60) + + @classmethod + def packages(cls, version, identity, content='example'): + directory = cls.root / ('packages-' + version + '-' + content) + directory.mkdir() + payload = 'debuglet-payload-' + identity + for name in ('debuglet', payload): + staging = directory / name + (staging / 'DEBIAN').mkdir(parents=True) + control = (f'Package: {name}\nVersion: {version}\nArchitecture: amd64\n' + 'Maintainer: Debuglet test \nDescription: apt fixture\n') + if name == 'debuglet': + control += f'Depends: {payload} (= {version})\n' + (staging / 'DEBIAN/control').write_text(control) + (staging / 'example').write_text(content) + # Release input filenames need not match their Debian control names. + filename = 'selector.deb' if name == 'debuglet' else 'payload.deb' + cls.run_command(['dpkg-deb', '--root-owner-group', '--build', str(staging), str(directory / filename)]) + return directory + + @classmethod + def sign(cls, directory): + cls.run_command(cls.gpg + ['--yes', '--local-user', cls.fingerprint, '--clearsign', + '--output', str(directory / apt.SUITE / 'InRelease'), + str(directory / apt.SUITE / 'Release')]) + + def copied(self, suffix): + directory = self.root / (self._testMethodName + '-' + suffix) + shutil.copytree(self.old, directory) + return directory + + def test_signed_release_and_all_retained_packages(self): + self.assertEqual(apt.verify(self.old, self.key, self.fingerprint)['packages'], 2) + upgraded = self.root / 'upgraded' + apt.build(self.new_packages, upgraded, self.old) + self.sign(upgraded) + self.assertEqual(apt.verify(upgraded, self.key, self.fingerprint)['packages'], 4) + for old_file in (self.old / apt.POOL).glob('*.deb'): + self.assertEqual(old_file.read_bytes(), (upgraded / apt.POOL / old_file.name).read_bytes()) + old_index = apt.regular(self.old / apt.SUITE / apt.INDEX) + self.assertEqual((upgraded / apt.SUITE / apt.INDEX.parent / 'by-hash/SHA256' / + apt.digest(old_index)).read_bytes(), old_index) + + def test_conflicting_retained_package_refused(self): + conflicting = self.packages('1.0.0-1', 'a' * 24, 'conflicting') + with self.assertRaisesRegex(ValueError, 'replace a retained package'): + apt.build(conflicting, self.root / 'conflict', self.old) + + def test_older_selector_refused(self): + older = self.packages('0.9.0-1', 'c' * 24) + with self.assertRaisesRegex(ValueError, 'older selector'): + apt.build(older, self.root / 'older', self.old) + + def test_wrong_fingerprint_and_wrong_key_refused(self): + with self.assertRaisesRegex(ValueError, 'fingerprint'): + apt.verify(self.old, self.key, '0' * 40) + self.run_command(self.gpg + ['--pinentry-mode', 'loopback', '--passphrase', '', + '--quick-generate-key', 'Wrong apt test ', + 'ed25519', 'sign', '0']) + records = self.run_command(self.gpg + ['--with-colons', '--list-keys', 'wrong@example.invalid']).decode().splitlines() + fingerprint = next(line.split(':')[9] for line in records if line.startswith('fpr:')) + key = self.root / 'wrong.asc' + key.write_bytes(self.run_command(self.gpg + ['--armor', '--export', fingerprint])) + with self.assertRaises(subprocess.CalledProcessError): + apt.verify(self.old, key, fingerprint) + + def test_each_hash_binding_refuses_modified_bytes(self): + targets = ['dists/stable/InRelease', str(apt.SUITE / apt.INDEX), + str(next((self.old / apt.POOL).glob('*.deb')).relative_to(self.old))] + for index, target in enumerate(targets): + with self.subTest(target=target): + directory = self.copied(str(index)) + path = directory / target + if target.endswith('InRelease'): + path.write_bytes(path.read_bytes().replace(b'Origin: Debuglet', b'Origin: Tampered')) + else: + path.write_bytes(path.read_bytes() + b'corruption') + with self.assertRaises((ValueError, subprocess.CalledProcessError)): + apt.verify(directory, self.key, self.fingerprint) + + def test_fetch_verifies_before_retaining_and_bootstrap_is_explicit(self): + class QuietHandler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(RepositoryTests.root), **kwargs) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), QuietHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + url = f'http://127.0.0.1:{server.server_port}' + try: + with self.assertRaises(ValueError): + apt.fetch(url + '/old', self.root / 'no-http', self.key, self.fingerprint) + result = apt.fetch(url + '/old', self.root / 'fetched', self.key, self.fingerprint, allow_local_http=True) + self.assertEqual(result['packages'], 2) + with self.assertRaisesRegex(ValueError, 'already exists'): + apt.fetch(url + '/old', self.root / 'bootstrap-existing', self.key, self.fingerprint, + bootstrap=True, allow_local_http=True) + result = apt.fetch(url + '/absent', self.root / 'bootstrap', self.key, self.fingerprint, + bootstrap=True, allow_local_http=True) + self.assertTrue(result['bootstrap']) + with self.assertRaises(OSError): + apt.fetch(url + '/absent', self.root / 'missing', self.key, self.fingerprint, allow_local_http=True) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +if __name__ == '__main__': + unittest.main() From 0f66c04cef1e38f8d1af2b1120dbe771f37eb89c Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 13:59:12 +0200 Subject: [PATCH 05/10] test: exercise signed apt installation and interrupted recovery --- deploy/ci/debian-check.py | 208 ++++++++++++++++++++++++++++ deploy/ci/debian-fixture.Dockerfile | 5 + deploy/ci/debian-prepare.py | 39 ++++++ scripts/ci-debian.sh | 39 ++++++ 4 files changed, 291 insertions(+) create mode 100644 deploy/ci/debian-check.py create mode 100644 deploy/ci/debian-fixture.Dockerfile create mode 100644 deploy/ci/debian-prepare.py create mode 100755 scripts/ci-debian.sh diff --git a/deploy/ci/debian-check.py b/deploy/ci/debian-check.py new file mode 100644 index 00000000..392fbee3 --- /dev/null +++ b/deploy/ci/debian-check.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Exercise apt's authentication and the installed Debian package lifecycle. + +The caller supplies a repository signed by a disposable test key, its exported +public key, a different exported key, and the package inventory. No network or +source toolchain is needed in the fresh Ubuntu 24.04 container. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import select +import shutil +import signal +import subprocess +import tempfile +import time + + +def run(*args, check=True): + result = subprocess.run(args, text=True, capture_output=True, timeout=120) + if check and result.returncode: + raise AssertionError(f"{args[0]} failed ({result.returncode}): {result.stdout}\n{result.stderr}") + return result + + +def event(check, **details): + print(json.dumps({"check": check, "passed": True, **details}), flush=True) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def interrupt_unpack(package, package_name, log): + """Kill the owned dpkg process group after its half-installed status event.""" + reader, writer = os.pipe() + process = None + buffer = b"" + observed = False + try: + with log.open("wb") as output: + process = subprocess.Popen(["dpkg", "--status-fd", str(writer), "--unpack", str(package)], + stdout=output, stderr=output, pass_fds=(writer,), start_new_session=True) + os.close(writer) + writer = None + deadline = time.monotonic() + 45 + while process.poll() is None and time.monotonic() < deadline: + if not select.select([reader], [], [], 1)[0]: + continue + chunk = os.read(reader, 4096) + if not chunk: + break + buffer += chunk + if f"status: {package_name}: half-installed".encode() in buffer: + os.killpg(process.pid, signal.SIGKILL) + observed = True + break + process.wait(timeout=15) + assert observed and process.returncode != 0, f"did not interrupt unpack: {buffer!r}\n{log.read_text()}" + finally: + os.close(reader) + if writer is not None: + os.close(writer) + if process is not None and process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=15) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repository", type=Path, required=True) + parser.add_argument("--key", type=Path, required=True) + parser.add_argument("--wrong-key", type=Path, required=True) + parser.add_argument("--inventory", type=Path, required=True) + args = parser.parse_args() + assert os.geteuid() == 0 and os.environ.get("DEBUGLET_DEBIAN_FIXTURE") == "1" + assert Path("/.dockerenv").exists(), "refusing to modify a non-container system" + release = dict(line.split("=", 1) for line in Path("/etc/os-release").read_text().splitlines() if "=" in line) + assert release["ID"].strip('"') == "ubuntu" and release["VERSION_ID"].strip('"') == "24.04" + assert run("dpkg", "--print-architecture").stdout.strip() == "amd64" + assert run("dpkg-query", "-W", "-f=${db:Status-Status}", "debuglet", check=False).stdout.strip() != "installed" + assert not Path("/etc/debuglet").exists() and not Path("/var/lib/debuglet").exists() + inventory = json.loads(args.inventory.read_text()) + payload_package = inventory["payload_package"] + payload_root = Path(inventory["payload_root"]) + assert payload_package.startswith("debuglet-payload-") + assert str(payload_root).startswith("/usr/lib/debuglet/debian/") + + with tempfile.TemporaryDirectory(prefix="debuglet-debian-") as directory: + work = Path(directory) + work.chmod(0o755) + repository = work / "repository" + shutil.copytree(args.repository, repository) + for path in [repository, *repository.rglob("*")]: + path.chmod(0o755 if path.is_dir() else 0o644) + key = work / "archive-key.asc" + source = work / "debuglet.sources" + source.write_text(f"Types: deb\nURIs: file:{repository}\nSuites: stable\nComponents: main\nArchitectures: amd64\nSigned-By: {key}\n") + source.chmod(0o644) + lists = work / "lists" + archives = work / "archives" + for path in (lists / "partial", archives / "partial"): + path.mkdir(parents=True) + options = ["-o", f"Dir::Etc::sourcelist={source}", "-o", "Dir::Etc::sourceparts=-", + "-o", f"Dir::State::lists={lists}", "-o", f"Dir::Cache::archives={archives}", + "-o", "APT::Get::List-Cleanup=0", "-o", "Acquire::Languages=none", + "-o", "Acquire::Retries=0", "-o", "APT::Update::Error-Mode=any"] + + def apt(*arguments, check=True): + return run("apt-get", *options, *arguments, check=check) + + def clear_lists(): + shutil.rmtree(lists) + (lists / "partial").mkdir(parents=True) + + def reject(label, *arguments, diagnostic): + result = apt(*arguments, check=False) + text = result.stdout + result.stderr + assert result.returncode != 0 and any(item in text for item in diagnostic), text + assert run("dpkg-query", "-W", "-f=${db:Status-Status}", "debuglet", check=False).stdout.strip() != "installed" + event(label, exit_code=result.returncode) + + shutil.copyfile(args.wrong_key, key) + key.chmod(0o644) + reject("wrong_scoped_key_rejected", "update", diagnostic=("NO_PUBKEY", "signature verification failed")) + shutil.copyfile(args.key, key) + clear_lists() + inrelease = repository / "dists/stable/InRelease" + authentic_release = inrelease.read_bytes() + inrelease.write_bytes(authentic_release.replace(b"Suite: stable", b"Suite: broken", 1)) + assert inrelease.read_bytes() != authentic_release, "test repository must declare Suite: stable" + reject("modified_signed_metadata_rejected", "update", diagnostic=("BADSIG", "invalid signature", "signatures were invalid")) + inrelease.write_bytes(authentic_release) + clear_lists() + indexes = {path: path.read_bytes() for path in (repository / "dists/stable/main/binary-amd64").rglob("*") if path.is_file()} + for path, contents in indexes.items(): + path.write_bytes(contents + b"modified index\n") + reject("modified_package_index_rejected", "update", diagnostic=("Hash Sum mismatch", "File has unexpected size")) + for path, contents in indexes.items(): + path.write_bytes(contents) + clear_lists() + apt("update") + event("signed_index_authenticated") + packages = sorted((repository / "pool").rglob("*.deb")) + assert packages, "repository has no Debian packages" + selected = next(path for path in packages if run("dpkg-deb", "-f", str(path), "Package").stdout.strip() == payload_package) + authentic_package = selected.read_bytes() + selected.write_bytes(authentic_package + b"tampered\n") + reject("modified_package_rejected", "--download-only", "-y", "install", "debuglet", diagnostic=("Hash Sum mismatch", "File has unexpected size")) + selected.write_bytes(authentic_package) + apt("-y", "install", "debuglet") + assert Path("/usr/bin/dbl").resolve() == payload_root / "bin/dbl" + identity = json.loads(run("/usr/bin/dbl", "--output", "json", "version").stdout) + assert identity["version"] == inventory["upstream_version"] and identity["revision"] == inventory["source_sha"] + assert not identity["modified"] + assert not list(Path("/etc/systemd/system").glob("debuglet-*.service")), "apt install activated a service" + event("fresh_apt_install", version=identity["version"], revision=identity["revision"], payload_root=str(payload_root)) + + state = Path("/var/lib/debuglet/debian-check") + config = Path("/etc/debuglet/debian-check.conf") + state.mkdir(parents=True, mode=0o700) + config.parent.mkdir(exist_ok=True) + config.write_text("preserve this operator configuration\n") + secret = state / "identity.key" + secret.write_text("owned fixture identity\n") + secret.chmod(0o600) + kept = {str(path): (digest(path), path.stat().st_uid, path.stat().st_gid, path.stat().st_mode) for path in (config, secret)} + # Reinstall the actual immutable payload, including repairing a damaged + # non-running member, without creating a new version or identity. + member = next(path for path in payload_root.rglob("*.wasm") if path.is_file()) + original_member = member.read_bytes() + member.write_bytes(b"damaged fixture member\n") + apt("-y", "--reinstall", "install", payload_package) + assert member.read_bytes() == original_member + event("same_payload_reinstall_repaired_member") + interrupt_unpack(selected, payload_package, work / "interrupted-unpack.log") + interrupted_status = run("dpkg-query", "-W", "-f=${db:Status-Status}", payload_package).stdout.strip() + # apt correctly refuses an interrupted dpkg journal until dpkg has + # processed it. A half-installed payload can then need reinstallation. + configured = run("dpkg", "--configure", "-a", check=False) + apt("-y", "--fix-broken", "--reinstall", "install", payload_package, "debuglet") + assert member.read_bytes() == original_member + assert run("dpkg-query", "-W", "-f=${db:Status-Status}", payload_package).stdout.strip() == "installed" + event("interrupted_payload_unpack_recovered", observed_status=interrupted_status, + configure_exit_code=configured.returncode) + # Model the documented dpkg interruption boundary: unpacked but not yet + # configured. Repeating apt install must finish configuration safely. + selector = next(path for path in packages if run("dpkg-deb", "-f", str(path), "Package").stdout.strip() == "debuglet") + run("dpkg", "--unpack", str(selector)) + assert run("dpkg-query", "-W", "-f=${db:Status-Status}", "debuglet").stdout.strip() == "unpacked" + apt("-y", "install", "debuglet") + assert run("dpkg-query", "-W", "-f=${db:Status-Status}", "debuglet").stdout.strip() == "installed" + run("/usr/bin/dbl", "--help") + event("unpacked_configuration_retry") + apt("-y", "purge", "debuglet", payload_package) + assert not payload_root.exists() and not Path("/usr/bin/dbl").exists() + assert kept == {str(path): (digest(path), path.stat().st_uid, path.stat().st_gid, path.stat().st_mode) for path in (config, secret)} + event("package_purge_preserved_operator_data") + secret.unlink() + state.rmdir() + config.unlink() + + +if __name__ == "__main__": + main() diff --git a/deploy/ci/debian-fixture.Dockerfile b/deploy/ci/debian-fixture.Dockerfile new file mode 100644 index 00000000..7181f880 --- /dev/null +++ b/deploy/ci/debian-fixture.Dockerfile @@ -0,0 +1,5 @@ +# The acceptance check records the resolved image ID alongside its result. +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends \ + adduser apt-utils ca-certificates gnupg python3 \ + && rm -rf /var/lib/apt/lists/* diff --git a/deploy/ci/debian-prepare.py b/deploy/ci/debian-prepare.py new file mode 100644 index 00000000..3a325149 --- /dev/null +++ b/deploy/ci/debian-prepare.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Create disposable repository trust material for the Debian acceptance check.""" + +import json +from pathlib import Path +import subprocess + + +def run(*args): + result = subprocess.run(args, text=True, capture_output=True, timeout=90) + if result.returncode: + raise AssertionError(f"{args[0]} failed ({result.returncode}): {result.stdout}\n{result.stderr}") + return result.stdout + + +def main(): + run("python3", "/source/tools/test_apt_repository.py") + work = Path("/work") + home = work / "gnupg" + home.mkdir(mode=0o700) + command = ["gpg", "--homedir", str(home), "--batch", "--yes"] + fingerprints = [] + for name, filename in (("Repository acceptance", "key.asc"), ("Wrong repository", "wrong-key.asc")): + run(*command, "--pinentry-mode", "loopback", "--passphrase", "", "--quick-generate-key", name, "rsa2048", "sign", "1d") + listing = run(*command, "--with-colons", "--list-keys", name) + fingerprint = next(line.split(":")[9] for line in listing.splitlines() if line.startswith("fpr:")) + (work / filename).write_text(run(*command, "--armor", "--export", fingerprint)) + fingerprints.append(fingerprint) + repository = work / "repository" + run("python3", "/source/tools/apt_repository.py", "build", "--packages", "/packages", "--out", str(repository)) + release = repository / "dists/stable/Release" + run(*command, "--local-user", fingerprints[0], "--clearsign", "--output", str(release.with_name("InRelease")), str(release)) + run(*command, "--local-user", fingerprints[0], "--armor", "--detach-sign", "--output", str(release.with_name("Release.gpg")), str(release)) + run("python3", "/source/tools/apt_repository.py", "verify", "--directory", str(repository), "--key", str(work / "key.asc"), "--fingerprint", fingerprints[0]) + print(json.dumps({"check": "disposable_repository_prepared", "passed": True, "fingerprint": fingerprints[0]}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci-debian.sh b/scripts/ci-debian.sh new file mode 100755 index 00000000..19f73e18 --- /dev/null +++ b/scripts/ci-debian.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Run real apt authentication and package installation in a fresh Ubuntu image. +set -euo pipefail +cd "$(dirname "$0")/.." +[[ $(uname -s) == Linux ]] || { echo 'Debian checks require a Linux Docker host.' >&2; exit 1; } +packages=$(realpath -e "${CI_DEBIAN_PACKAGE_DIR:-${CI_PACKAGE_DIR:-.cache/ci/packages}/debian}") +[[ -f "$packages/deb-packages.json" ]] || { echo 'Build the Debian packages first.' >&2; exit 1; } +mkdir -p .cache/ci +work=$(mktemp -d "$PWD/.cache/ci/debian-fixture.XXXXXXXX") +image=${DEBUGLET_DEBIAN_TEST_IMAGE:-debuglet-debian-test:ubuntu24.04} +name="debuglet-debian-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-$$" +cleanup() { + local status=$? + docker rm -f "$name-prepare" "$name-check" >/dev/null 2>&1 || true + rm -rf -- "$work" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +if [[ -z ${DEBUGLET_DEBIAN_TEST_IMAGE:-} ]]; then + docker build -t "$image" -f deploy/ci/debian-fixture.Dockerfile deploy/ci +fi +docker image inspect --format '{{.Id}}' "$image" > .cache/ci/debian-image.txt +docker run --rm --name "$name-prepare" --network none --cpus=2 --memory=1g --pids-limit=128 \ + --user "$(id -u):$(id -g)" --cap-drop=ALL --security-opt=no-new-privileges \ + --mount "type=bind,src=$PWD,dst=/source,readonly" \ + --mount "type=bind,src=$packages,dst=/packages,readonly" \ + --mount "type=bind,src=$work,dst=/work" \ + --entrypoint python3 "$image" /source/deploy/ci/debian-prepare.py \ + > .cache/ci/debian-prepare.jsonl +docker run --rm --name "$name-check" --network none --cpus=2 --memory=1g --pids-limit=128 \ + --cap-add=SYS_PTRACE --security-opt=no-new-privileges --env DEBUGLET_DEBIAN_FIXTURE=1 \ + --mount "type=bind,src=$PWD,dst=/source,readonly" \ + --mount "type=bind,src=$packages,dst=/packages,readonly" \ + --mount "type=bind,src=$work,dst=/fixture,readonly" \ + --entrypoint python3 "$image" /source/deploy/ci/debian-check.py \ + --repository /fixture/repository --key /fixture/key.asc --wrong-key /fixture/wrong-key.asc \ + --inventory /packages/deb-packages.json | tee .cache/ci/debian-check.jsonl From 3a0bd47b0ae487087f5e66ddc8c1ee933bf6a7c1 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 13:59:53 +0200 Subject: [PATCH 06/10] Select console with apt setup and managed update instructions --- configs/release-compatibility.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/release-compatibility.json b/configs/release-compatibility.json index cfc1343e..c227305f 100644 --- a/configs/release-compatibility.json +++ b/configs/release-compatibility.json @@ -1,4 +1,4 @@ { "console_repository": "https://gitlab.inf.ethz.ch/OU-PERRIG/yimin/debuglet/debuglet-dashboard", - "console_revision": "5312f70a1ebd9e78630ac6ffc4ae67383dcb4e14" + "console_revision": "a117af96c73d712b58ed3a16bbbe4fb15bf9ac3d" } From f47bc5196620b06c106206d93c6f09fdb805fea6 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 14:10:52 +0200 Subject: [PATCH 07/10] Clarify optional Pages deployment review --- docs/operations/apt-repository.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/operations/apt-repository.md b/docs/operations/apt-repository.md index 7f7c4f1b..d345d0b1 100644 --- a/docs/operations/apt-repository.md +++ b/docs/operations/apt-repository.md @@ -62,9 +62,11 @@ metadata. different format. 3. Keep `release-signing` protected by required independent reviewers with prevent-self-review enabled, and restrict it to the approved version tags. - Create or review the `github-pages` environment with an independent reviewer - and deployment rules allowing only those release tags. Do not broaden tag - creation or environment rules to make a maintainer's blocked run pass. + Create or review the `github-pages` environment with deployment rules allowing + only those release tags and no signing secret. Additional Pages reviewers are + optional operator policy; the required independent signing approval remains + in `release-signing`. Do not broaden tag creation or environment rules to make + a maintainer's blocked run pass. 4. Enable GitHub Pages for this repository with **GitHub Actions** as the source, configure the desired domain if applicable, and enforce HTTPS. This workflow owns the entire Pages site; do not share that destination with another Pages @@ -105,8 +107,8 @@ For subsequent publications leave bootstrap false. The workflow downloads the existing InRelease and verifies it against the configured OpenPGP key, then verifies the hashes of both indexes and every retained package. Missing, changed or untrusted content stops publication. It then indexes the old and new packages, -requests the signing approval, and deploys the signed site after the Pages -environment's approval. Publication runs are serialized. +requests the signing approval, and deploys the signed site under the configured +Pages environment policy. Publication runs are serialized. The workflow saves a signed site artifact for 90 days, but Actions retention is not a durable archive. Keep an independent backup of each approved signed site From 6afd1802e083a27de88fc39535b976327936bb27 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 14:17:38 +0200 Subject: [PATCH 08/10] Check Debian repository regressions in package CI and retain console rollback payloads --- configs/release-compatibility.json | 2 +- scripts/ci-debian.sh | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/configs/release-compatibility.json b/configs/release-compatibility.json index c227305f..d00847f2 100644 --- a/configs/release-compatibility.json +++ b/configs/release-compatibility.json @@ -1,4 +1,4 @@ { "console_repository": "https://gitlab.inf.ethz.ch/OU-PERRIG/yimin/debuglet/debuglet-dashboard", - "console_revision": "a117af96c73d712b58ed3a16bbbe4fb15bf9ac3d" + "console_revision": "f5dd418e04ea0defd2b14ac97ab0c2de648217d5" } diff --git a/scripts/ci-debian.sh b/scripts/ci-debian.sh index 19f73e18..5f7702e3 100755 --- a/scripts/ci-debian.sh +++ b/scripts/ci-debian.sh @@ -27,7 +27,12 @@ docker run --rm --name "$name-prepare" --network none --cpus=2 --memory=1g --pid --mount "type=bind,src=$PWD,dst=/source,readonly" \ --mount "type=bind,src=$packages,dst=/packages,readonly" \ --mount "type=bind,src=$work,dst=/work" \ - --entrypoint python3 "$image" /source/deploy/ci/debian-prepare.py \ + --entrypoint sh "$image" -ec ' + command -v apt-ftparchive dpkg-deb gpg gpgv >/dev/null + python3 -m unittest discover -s /source/deploy/debian -p test_package.py -v >&2 + python3 -m unittest discover -s /source/tools -p test_apt_repository.py -v >&2 + exec python3 /source/deploy/ci/debian-prepare.py + ' \ > .cache/ci/debian-prepare.jsonl docker run --rm --name "$name-check" --network none --cpus=2 --memory=1g --pids-limit=128 \ --cap-add=SYS_PTRACE --security-opt=no-new-privileges --env DEBUGLET_DEBIAN_FIXTURE=1 \ From c057f4305cfb8c34cce015830f879eb6fe274be5 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 14:31:29 +0200 Subject: [PATCH 09/10] Report running executor version after package replacement --- internal/executor/hello_policy_test.go | 17 +++++++++++++++++ internal/executor/rpc_handlers.go | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/internal/executor/hello_policy_test.go b/internal/executor/hello_policy_test.go index e4bc8995..b61444a7 100644 --- a/internal/executor/hello_policy_test.go +++ b/internal/executor/hello_policy_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/netsec-ethz/debuglet/internal/buildinfo" "github.com/netsec-ethz/debuglet/internal/executor/debuglet/netpolicy" "github.com/netsec-ethz/debuglet/internal/executor/ratelimit" ) @@ -63,3 +64,19 @@ func TestHelloAdvertisesICMPFromTheNetworkPolicy(t *testing.T) { }) } } + +func TestHelloReportsRunningVersionAfterPackageReplacement(t *testing.T) { + cfg := fixtureConfig() + cfg.Identity.Version = "v0.0.0-dev.previous" + e := newFixtureExecutor(t, cfg, nil, newFixtureMemoryStorage(t)) + resp, err := e.OnHello(t.Context(), nil) + if err != nil { + t.Fatal(err) + } + if resp.GetVersion() != buildinfo.Version { + t.Fatalf("reported version=%q, want running build %q", resp.GetVersion(), buildinfo.Version) + } + if cfg.Identity.Version != "v0.0.0-dev.previous" { + t.Fatalf("stored version changed to %q", cfg.Identity.Version) + } +} diff --git a/internal/executor/rpc_handlers.go b/internal/executor/rpc_handlers.go index bb29b2db..95c473a4 100644 --- a/internal/executor/rpc_handlers.go +++ b/internal/executor/rpc_handlers.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "github.com/netsec-ethz/debuglet/internal/bitrate" + "github.com/netsec-ethz/debuglet/internal/buildinfo" "github.com/netsec-ethz/debuglet/internal/controlsession" "github.com/netsec-ethz/debuglet/internal/daemonlog" "github.com/netsec-ethz/debuglet/internal/executor/debuglet/netpolicy" @@ -65,7 +66,7 @@ func (e *Executor) OnHello(ctx context.Context, req *pb.HelloRequest) (*pb.Hello ExecutorId: e.cfg.Identity.ExecutorID, BandwidthVersion: 2, EgressBudgetVersion: 1, - Version: e.cfg.Identity.Version, + Version: buildinfo.Version, Capabilities: capabilities, VantagePoint: vantage, // The dispatcher records the address it observes on the control From 8ab21302755047c76687fbea0224fb8f7b9caa94 Mon Sep 17 00:00:00 2001 From: TheodorAdrienIsaak Mattli Date: Thu, 15 Oct 2026 14:43:48 +0200 Subject: [PATCH 10/10] Synchronize guest read-reset transport fixture after connect --- pkg/debuglet/io_integration_test.go | 10 +++++++++- pkg/debuglet/testdata/recoverable_io/main.go | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/debuglet/io_integration_test.go b/pkg/debuglet/io_integration_test.go index d97330bd..a5eb8615 100644 --- a/pkg/debuglet/io_integration_test.go +++ b/pkg/debuglet/io_integration_test.go @@ -161,7 +161,15 @@ func TestRecoverableGuestRealTransports(t *testing.T) { _, _ = c.Write([]byte("PONG")) } }) - } else if mode == "reset" || mode == "connect-reset" { + } else if mode == "reset" { + addr = startTCPTarget(t, func(c net.Conn) { + // Reset only after the guest has completed Dial and sent its marker. + var marker [1]byte + if _, err := io.ReadFull(c, marker[:]); err == nil && marker[0] == 'R' { + _ = c.(*net.TCPConn).SetLinger(0) + } + }) + } else if mode == "connect-reset" { addr = startTCPTarget(t, func(c net.Conn) { _ = c.(*net.TCPConn).SetLinger(0) }) } else if mode == "connect-timeout" { addr = startTCPTarget(t, func(c net.Conn) { _, _ = io.Copy(io.Discard, c) }) diff --git a/pkg/debuglet/testdata/recoverable_io/main.go b/pkg/debuglet/testdata/recoverable_io/main.go index d251bbea..ec9c6d4f 100644 --- a/pkg/debuglet/testdata/recoverable_io/main.go +++ b/pkg/debuglet/testdata/recoverable_io/main.go @@ -78,7 +78,9 @@ func main() { n, err = io.ReadFull(s, buf[:4]) require(n == 4 && err == nil && string(buf[:n]) == "PONG", n, err) case "reset": - n, err := s.Read(buf) + n, err := s.Write([]byte("R")) + require(n == 1 && err == nil, n, err) + n, err = s.Read(buf) require(n == 0 && errors.Is(err, debuglet.ErrReset), n, err) default: panic("unknown mode")