From 9571cd1314c710bf6a9b1247842f26b2ed0c6f95 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 17:06:42 +0000 Subject: [PATCH 1/4] scripts: CS-RG qualification harness (CSRG-U6) scripts/qualify-devguard.py runs the CSRG-U6 qualification on the macOS host: candidate records the exact combination, matrix runs the claimed modes against the installed DevGuard service, faults and authority inject failures against DevGuard's fixture authority, control measures the control path under DG-1's load, and verify recomputes a sealed run's verdicts from its raw files. It drives codespace-mcp over MCP stdio with its own small client, reads DevGuard's journal read-only, gives every child a minimal environment, and never reads, prints or passes a credential: the gateway is given the credential file's path. qualify_analysis.py holds the verdict rules as pure functions; Python 3.9 with no third-party packages. Co-Authored-By: Claude Opus 5.5 (1M context) --- scripts/qualify-devguard.py | 2277 ++++++++++++++++++++++++ scripts/qualify_analysis.py | 358 ++++ scripts/qualify_host.py | 913 ++++++++++ scripts/tests/test_qualify_devguard.py | 724 ++++++++ 4 files changed, 4272 insertions(+) create mode 100644 scripts/qualify-devguard.py create mode 100644 scripts/qualify_analysis.py create mode 100644 scripts/qualify_host.py create mode 100644 scripts/tests/test_qualify_devguard.py diff --git a/scripts/qualify-devguard.py b/scripts/qualify-devguard.py new file mode 100644 index 0000000..9b6631b --- /dev/null +++ b/scripts/qualify-devguard.py @@ -0,0 +1,2277 @@ +#!/usr/bin/env python3 +"""CSRG-U6 qualification of CodeSpace with DevGuard; docs/cs-rg-qualification.md is the protocol. + + candidate record the exact combination under test + matrix the claimed modes against the installed DevGuard service, each launch checked + against DevGuard's own record of its attempt + faults the cases that would leave the installed service with Suspect or charged work, + against DevGuard's fixture authority at CodeSpace's pin + control control-path latency under DG-1's development load, against the installed service + authority control-path latency while the fixture authority stalls, pauses, restarts and stops + verify check a sealed run and recompute its verdicts from its raw evidence + +Every suite but verify writes a new evidence directory and seals it; verify writes nothing. +Nothing here edits DevGuard's configuration, restarts its service or reads its credential: the +gateway is given that file's path and reads it itself, and the fault cases use DevGuard's fixture +authority and throwaway credentials in their own scratch directories. Exit status: 0 passed or +qualified, 1 failed, 2 incomplete or inconclusive, 3 the harness failed, 130 interrupted; verify +exits 0 when the sealed run checks out and 1 when it does not. +""" +import argparse +import hashlib +import importlib.util +import itertools +import json +import os +from pathlib import Path +import platform +import re +import secrets +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import traceback +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import qualify_analysis as analysis # noqa: E402 +from qualify_analysis import check # noqa: E402 +import qualify_host as host # noqa: E402 +from qualify_host import ToolError # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +PINNED_RUST = '1.95.0' +EXIT = {'passed': 0, 'qualified': 0, 'failed': 1, 'incomplete': 2, 'inconclusive': 2} +EXIT_CRASH = 3 +IO = (False, True) +MODES = ('in-process', 'uds') +# What a matrix execution asks for; it runs one process at a time, never a descendant. +MATRIX_REQUEST = {'cpu_milli': 50, 'memory_bytes': 64 << 20, 'tasks': 8} +KERNEL_MINIMUM = {'cpu': 'accounted', 'memory': 'accounted', 'pids': 'kernel'} +MATRIX_TIMEOUT_S = 15 +APPLIED = {'cpu': 'applied', 'memory': 'applied', 'pids': 'applied'} +# The claimed modes (docs/cs-rg-qualification.md, "Supported modes"). +CLAIMS = [ + {'platform': 'macos', 'runner': 'in-process', 'io': ['pipe', 'pty'], 'participation': 'required', 'claim': 'qualify'}, + {'platform': 'macos', 'runner': 'uds --runtime-bin', 'io': ['pipe', 'pty'], 'participation': 'required', 'claim': 'qualify'}, + {'platform': 'macos', 'runner': 'uds --runner-socket', 'io': ['pipe', 'pty'], 'participation': 'required', 'claim': 'unsupported'}, + {'platform': 'macos', 'runner': 'any', 'io': ['pipe', 'pty'], 'participation': 'required under --devguard status', 'claim': 'unsupported'}, + {'platform': 'macos', 'runner': 'supported', 'io': ['pipe', 'pty'], 'participation': 'required without a launch helper', 'claim': 'unsupported'}, + {'platform': 'macos', 'runner': 'supported', 'io': ['pipe', 'pty'], 'participation': 'off and required in one gateway', 'claim': 'qualify'}, + {'platform': 'macos, linux', 'runner': 'all', 'io': ['pipe', 'pty'], 'participation': 'off', 'claim': 'existing behavior'}, + {'platform': 'linux', 'runner': 'all', 'io': ['pipe', 'pty'], 'participation': 'required', 'claim': 'unsupported until DG-LINUX'}, +] + + +# ---------------------------------------------------------------- what a run starts + +class Live: + """What a run started, settled however it ends: every process a gateway started is ended and + waited for before the gateway's input closes, then workers and fixture authorities stop, and + DG-1's load owners are asked to settle their scopes.""" + + def __init__(self): + self.gateways, self.fixtures, self.workers = [], [], [] + self.measure = None + self.reports = [] + + def settle(self): + for number in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(number, signal.SIG_IGN) + if self.measure is not None: + self.measure.settle_live() + for gateway in reversed(self.gateways): + if gateway.process.poll() is None: + left = gateway.settle(limit=20) + self.reports.append({'gateway': gateway.pid, 'left_running': left, **gateway.close()}) + for worker in self.workers: + if worker.poll() is None: + worker.terminate() + try: + worker.wait(timeout=10) + except subprocess.TimeoutExpired: + worker.kill() + worker.wait() + for fixture in self.fixtures: + fixture.stop() + return self.reports + + +def scratch(): + """A short private directory for sockets, workspaces and runner state: Unix socket paths are + limited to about 100 bytes.""" + return Path(tempfile.mkdtemp(prefix='csq-', dir='/private/tmp' if platform.system() == 'Darwin' else None)) + + +def git(*args, root=ROOT): + return subprocess.run(['git', '-C', str(root), *args], capture_output=True, text=True, check=True).stdout.strip() + + +def gone(pid, limit=10.0): + """Whether a process is gone within `limit` seconds.""" + deadline = time.monotonic() + limit + while True: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + except PermissionError: + pass + if time.monotonic() >= deadline: + return False + time.sleep(0.05) + + +# ---------------------------------------------------------------- candidate + +def documentation(path): + return path.startswith(('docs/', 'docs-site/')) or path.endswith('.md') + + +def code_tree(root=ROOT, commit='HEAD'): + """A hash of a commit's tree without documentation, so that a documentation-only commit keeps + the candidate; the Codex gitlink is part of it.""" + listing = git('ls-tree', '-r', '--full-tree', commit, root=root) + kept = [line for line in listing.splitlines() if not documentation(line.split('\t', 1)[1])] + return hashlib.sha256(('\n'.join(kept) + '\n').encode()).hexdigest() + + +def devguard_lock(root=ROOT): + """The `devguard-*` packages of Cargo.lock: the DevGuard client pin.""" + found = {} + for block in (root / 'Cargo.lock').read_text().split('[[package]]'): + name = re.search(r'^name = "([^"]+)"', block, re.M) + if name and name[1].startswith('devguard'): + version = re.search(r'^version = "([^"]+)"', block, re.M) + source = re.search(r'^source = "([^"]+)"', block, re.M) + found[name[1]] = {'version': version[1] if version else None, 'source': source[1] if source else None} + return found + + +SETTING_KEYS = ('role', 'generation', 'max_instances', 'cpu_milli', 'memory_bytes', 'tasks') + + +def consumer_settings(text, consumer=host.CONSUMER): + """A consumer's role, generation, instance limit and control reservation from host.toml. Only + these keys are read; a credential digest is never returned.""" + found, section = {}, None + prefix = f'consumers.{consumer}' + for raw in text.splitlines(): + line = raw.split('#', 1)[0].strip() + header = re.fullmatch(r'\[\s*([^\]]+?)\s*\]', line) + if header: + section = header[1].replace('"', '').replace("'", '').replace(' ', '') + continue + if section not in (prefix, prefix + '.control_reservation') or '=' not in line: + continue + key, value = (part.strip() for part in line.split('=', 1)) + if section == prefix and key == 'control_reservation' and value.startswith('{'): + for inner in re.finditer(r'(\w+)\s*=\s*(\d+)', value): + if inner[1] in SETTING_KEYS: + found.setdefault('control_reservation', {})[inner[1]] = int(inner[2]) + elif key in SETTING_KEYS: + target = found.setdefault('control_reservation', {}) if section != prefix else found + target[key] = int(value) if value.isdigit() else value.strip('"\'') + return found + + +def binary(path, version=False): + path = Path(path).resolve() + entry = {'path': str(path), 'sha256': host.sha256(path), 'bytes': path.stat().st_size} + if version: + entry['version'] = host.run_text(path, '--version') + return entry + + +def toolchain(): + found = {'rustc': host.run_text('rustc', '-vV'), 'cargo': host.run_text('cargo', '-V'), + 'python': sys.version.split()[0]} + if platform.system() == 'Darwin': + found['xcode'] = host.run_text('xcodebuild', '-version') + found['clt'] = host.run_text('xcode-select', '-p') + return found + + +def codex_check(): + result = subprocess.run([sys.executable, str(ROOT / 'scripts/codex_patch.py'), 'check'], capture_output=True, + text=True, cwd=ROOT) + return {'returncode': result.returncode, 'output': (result.stdout + result.stderr).strip()[-2000:]} + + +def policy(release, args): + """The service's policy as DevGuard's promotion compares it, and the codespace consumer.""" + config = Path(args.host_config) + found = {'host_toml_sha256': host.sha256(config) if config.exists() else None, 'doctor': release.doctor(), + 'consumer': consumer_settings(config.read_text()) if config.exists() else None} + qualification = host.SUPPORT / 'qualifications' / f'{release.id}.json' + if qualification.exists(): + record = json.loads(qualification.read_text()) + measured = record.get('policy') or {} + doctor = measured.get('doctor') or {} + found['dg1_qualification'] = { + 'path': str(qualification), 'sha256': host.sha256(qualification), + 'combinations': {name: entry.get('verdict') for name, entry in (record.get('combinations') or {}).items()}, + 'same_policy': (measured.get('host_toml_sha256') == found['host_toml_sha256'] + and doctor.get('configuration_fingerprint') == found['doctor']['configuration_fingerprint'] + and doctor.get('capabilities') == found['doctor']['capabilities'])} + else: + found['dg1_qualification'] = None + return found + + +def dirty(root=ROOT): + """Whether tracked files differ from the commit; untracked files do not change a build. The + Codex submodule's files hold the recorded patch on the qualification host, so its contents + are left to the Codex check; a submodule at another commit still counts.""" + return bool(git('status', '--porcelain', '--untracked-files=no', '--ignore-submodules=dirty', root=root)) + + +def candidate_problems(record): + """What keeps a recorded candidate from being the frozen one, from its record alone.""" + problems = [] + if record['codespace']['dirty']: + problems.append('the CodeSpace checkout has uncommitted changes') + if record['codespace']['codex_patch']['returncode']: + problems.append('the Codex checkout is not the recorded patched tree') + if not record['release']['state']['runs_release']: + problems.append('the service does not run the release') + if not all(entry['matches'] for entry in record['release']['artifacts'].values()): + problems.append('an installed binary differs from the release manifest') + if not record['policy']['doctor'].get('satisfied'): + problems.append('devguard doctor is not satisfied') + if (record['policy']['consumer'] or {}).get('role') != 'control_service': + problems.append('host.toml has no codespace control-service consumer') + if not (record['toolchain']['rustc'] or '').startswith(f'rustc {PINNED_RUST} '): + problems.append(f'Rust {PINNED_RUST} is not first in PATH') + return problems + + +def candidate(args, live, evidence): + release = host.Release(args.release) + record = { + 'schema': 'codespace-qualification-candidate/v1', 'recorded_unix': time.time(), + 'codespace': {'commit': git('rev-parse', 'HEAD'), 'dirty': dirty(), + 'code_tree_sha256': code_tree(), 'devguard_lock': devguard_lock(), + 'codex_gitlink': git('rev-parse', 'HEAD:third_party/codex'), 'codex_patch': codex_check()}, + 'binaries': {'codespace-mcp': binary(args.mcp_bin, version=True), + 'codespace-codex-runtime': binary(args.runtime_bin)}, + 'toolchain': toolchain(), + 'release': {'id': release.id, 'directory': str(release.directory), **release.artifacts(), + 'state': release.state()}, + 'policy': policy(release, args), + 'work_capacity': host.work_capacity(args.service_log), 'pressure': host.pressure_state(args.service_log), + 'host': host.host_facts(), 'claims': CLAIMS} + for name in ('fixture_bin', 'fixture_helper', 'qualify_bin'): + if getattr(args, name, None): + record['binaries'][name.replace('_', '-')] = binary(getattr(args, name)) + evidence.write_json('candidate.json', record) + problems = candidate_problems(record) + return {'verdict': 'failed' if problems else 'passed', 'problems': problems} + + +# ---------------------------------------------------------------- a runner mode's gateways + +class Bench: + """One runner mode's gateways against one authority, sharing a workspace registry and an + evidence prefix. `authority` names the socket, the credential file (its path only), the + launch helper, the consumer and the journal to check against; a fault case's bench also has + its fixture authority.""" + + def __init__(self, args, live, evidence, writer, mode, work, workspaces, authority, prefix=None, fixture=None): + self.args, self.live, self.evidence, self.writer, self.mode = args, live, evidence, writer, mode + self.work, self.authority, self.fixture = work, authority, fixture + self.prefix = prefix or mode + self.database = authority['journal'] + self.roots = {name: Path(value[0]) for name, value in workspaces.items()} + self.requests = {name: (value[1] or {}).get('request') for name, value in workspaces.items()} + self.config = evidence.file(f'{self.prefix}/registry.json') + self.config.write_text(json.dumps(host.registry(workspaces), indent=2) + '\n') + (work / 'runner').mkdir(mode=0o700, exist_ok=True) + self.count = 0 + self.owner_pid = None + self.gateways = [] + + def gateway(self, name, mode='register', helper='release', socket=None, timeout_s=None, runner_socket=None, + credential_file=None): + self.count += 1 + authority = self.authority + flags = host.runner_flags(self.mode, self.args.runtime_bin, self.work / 'runner', runner_socket) + helper = authority['helper'] if helper == 'release' else helper + flags += host.devguard_flags(mode, socket or authority['socket'], credential_file or authority['credential_file'], + helper, authority['consumer'], authority['generation']) + env = {'CODESPACE_CONFIG': str(self.config), + 'CODESPACE_OPERATIONS_DB': str(self.work / f'operations-{self.count}.sqlite')} + if timeout_s: + env['CODESPACE_PROCESS_TIMEOUT_SECS'] = str(timeout_s) + prefix = f'{self.prefix}/{self.count:02d}-{name}' + transcript = self.evidence.file(prefix + '.mcp.jsonl') + gateway = host.Gateway(self.args.mcp_bin, flags, host.minimal_env(self.work, env), + self.evidence.file(prefix + '.gateway.log'), + record=lambda row: self.writer.write(transcript, row)) + self.live.gateways.append(gateway) + self.gateways.append(gateway) + self.evidence.append(f'{self.prefix}/gateways.jsonl', {'name': name, 'pid': gateway.pid, 'argv': gateway.argv, + 'timeout_s': timeout_s, 'started_unix': time.time()}) + gateway.initialize() + return gateway + + def registration(self, gateway, workspace): + """workspace_info's resource authority, remembering the registered owner's PID.""" + info, latency = gateway.call('workspace_info', {'workspace_id': workspace}) + authority = info.get('resource_authority') or {} + pid = (authority.get('registration') or {}).get('pid') + if pid: + self.owner_pid = pid + return authority, latency + + def close(self, gateway): + """Settle and close a gateway, then wait for its execution owner to be gone, so that the + next owner's registration finds the consumer's one instance free.""" + left = gateway.settle() + closed = gateway.close() + owner_gone = gone(self.owner_pid) if self.owner_pid and self.owner_pid != gateway.pid else True + self.evidence.append(f'{self.prefix}/gateways.jsonl', {'closed': gateway.pid, 'left_running': left, + 'owner_gone': owner_gone, **closed}) + return left, closed, owner_gone + + def teardown(self): + """Close every gateway still running, stop the fixture authority and remove the work + directory, whatever state a case left them in.""" + for gateway in self.gateways: + if gateway.process.poll() is None: + try: + self.close(gateway) + except Exception as error: # noqa: BLE001 - teardown goes on regardless + self.evidence.append(f'{self.prefix}/gateways.jsonl', {'teardown_error': str(error)}) + gateway.close(limit=10) + if self.fixture is not None: + self.fixture.stop() + shutil.rmtree(self.work, ignore_errors=True) + + +def run_scenario(results, evidence, mode, io, name, function, *args): + """Run one scenario and keep its checks; a scenario that breaks is a failed check.""" + began = time.time() + try: + checks, detail = function(*args) + except Exception as error: # noqa: BLE001 - every broken scenario is evidence + checks = [check('the scenario completed', False, error=f'{type(error).__name__}: {error}', + trace=traceback.format_exc()[-3000:])] + detail = {} + entry = {'scenario': name, 'mode': mode, 'io': io, 'verdict': analysis.scenario_verdict(checks), + 'checks': checks, 'detail': detail, 'started_unix': began, 'ended_unix': time.time()} + results.append(entry) + evidence.append('scenarios.jsonl', entry) + print(f"qualify: {mode} {io or '-'} {name}: {entry['verdict']}", flush=True) + return entry + + +def io_name(tty): + return 'pty' if tty else 'pipe' + + +def quantities(request): + return {key: request[key] for key in ('cpu_milli', 'memory_bytes', 'tasks')} + + +def prefixed(label, checks): + return [dict(item, check=f"{label}: {item['check']}") for item in checks] + + +def account_checks(bench, gateway, process_id, request, limit=60.0): + """DevGuard's own record of one launched execution: the attempt the gateway logged reserved + exactly the workspace's request, applied every resource to the scope rooted at the launched + process, and was released once the scope ended, leaving nothing charged.""" + entry = gateway.execution(process_id) + checks = [check('the gateway logged a governed launch', bool(entry and entry.get('launched')), entry=entry)] + if not (entry and entry.get('launched')): + return checks, None + found, _ = host.settle_attempts([entry['attempt_id']], bench.database, limit, bench.authority['consumer']) + view = found.get(entry['attempt_id']) + checks.append(check('DevGuard recorded the attempt', view is not None, attempt_id=entry['attempt_id'])) + if view is None: + return checks, None + wanted = quantities(request) + checks += [ + check('it reserved and applied exactly the workspace request', + view['reserved'] == wanted and view['applied_quantities'] == wanted, + reserved=view['reserved'], applied=view['applied_quantities'], requested=wanted), + check('every resource was applied', view['applied'] == APPLIED, applied=view['applied']), + check('the scope root is the launched process', + view['scope_root_pid'] is not None and view['scope_root_pid'] == entry.get('scope_root_pid'), + journal=view['scope_root_pid'], logged=entry.get('scope_root_pid')), + check('released once its scope ended, holding nothing', + view['phase'] == 'released' and view['release_reason'] == 'scope_terminated' and not view['charged'], + phase=view['phase'], release_reason=view['release_reason'], charged=view['charged']), + check('its tracking stayed complete', view['tracking_lost'] is False, tracking_lost=view['tracking_lost'])] + return checks, dict(view, attempt_id=entry['attempt_id']) + + +def refusal_checks(bench, result, code, phase, denial=None): + """A refused exec_command: CodeSpace's code, and the attempt it names as DevGuard recorded it, + holding nothing. `phase` None means no attempt may exist; 'absent' that DevGuard has none.""" + checks = [check(f'refused with {code}', result['outcome'] == 'refused' and result.get('code') == code, + outcome=result['outcome'], code=result.get('code'), message=result.get('message'), + latency_ms=result.get('latency_ms'))] + attempt = result.get('attempt_id') + if phase is None: + checks.append(check('refused before any attempt existed', attempt is None, attempt_id=attempt)) + return checks + if phase == 'absent': + found = host.journal_attempts([attempt], bench.database, bench.authority['consumer']) if attempt else {} + checks.append(check('DevGuard holds no attempt for it', not found, attempt_id=attempt)) + return checks + checks.append(check('the refusal names its attempt', attempt is not None, message=result.get('message'))) + if attempt: + found, _ = host.settle_attempts([attempt], bench.database, 30, bench.authority['consumer']) + view = found.get(attempt) + checks.append(check(f'DevGuard recorded it {phase}, holding nothing', + view is not None and view['phase'] == phase and not view['charged'] + and (denial is None or view['denial'] == denial), + view=view, attempt_id=attempt)) + return checks + + +def started_checks(label, started): + return check(f'{label}: started and confirmed', started['outcome'] == 'started' and started.get('dispatch') == 'confirmed', + started=started) + + +# ---------------------------------------------------------------- matrix scenarios (installed service) +# Every command here runs one process, or children its root waits for: none survives its root, +# leaves its group or is reaped unobserved, so no attempt can stay Suspect on the owner's service. + +def scenario_registration(bench, gateway): + authority, latency = bench.registration(gateway, 'gov') + registration = authority.get('registration') or {} + pid = registration.get('pid') + owner = 'in_process' if bench.mode == 'in-process' else 'worker' + owner_ok = pid == gateway.pid if bench.mode == 'in-process' else pid in host.children(gateway.pid) + instances = host.journal_instances(bench.authority['consumer'], bench.database) + active = [entry for entry in instances if entry['state'] == 'active'] + checks = [ + check('participation is registration', authority.get('participation') == 'registration', authority=authority), + check('the authority is available', authority.get('state') == 'available'), + check(f'the {owner} owner is registered', registration.get('owner') == owner and registration.get('state') == 'registered', + registration=registration), + check('it governs executions', authority.get('governs_execution') is True), + check('the registered process is the owner', owner_ok, pid=pid, gateway=gateway.pid), + check("DevGuard's one active instance is that process", len(active) == 1 and active[0]['pid'] == pid, + instances=instances)] + return checks, {'workspace_info_ms': latency, 'resource_authority': authority, 'instances': instances} + + +def scenario_exits(bench, gateway, tty): + checks, detail = [], {} + # The pinned Codex PTY reports a signal death as exit code 1 (docs/agent-integration.md). + death = {'termination': 'exited', 'exit_code': 1} if tty else {'termination': 'signaled', 'signal': 15} + for label, command, expected in (('exit 0', ['true'], {'termination': 'exited', 'exit_code': 0}), + ('exit 3', ['sh', '-c', 'exit 3'], {'termination': 'exited', 'exit_code': 3}), + ('death by SIGTERM', ['sh', '-c', 'kill -TERM $$'], death)): + started = gateway.exec('gov', command, tty=tty) + checks.append(started_checks(label, started)) + if started['outcome'] != 'started': + continue + status = gateway.wait_exited(started['process_id']) + observed = {'termination': (status or {}).get('termination'), 'exit_code': (status or {}).get('exit_code'), + 'signal': ((status or {}).get('signal') or {}).get('number')} + checks.append(check(f'{label}: reported as expected', all(observed[key] == value for key, value in expected.items()), + observed=observed, expected=expected)) + account, view = account_checks(bench, gateway, started['process_id'], MATRIX_REQUEST) + checks += prefixed(label, account) + detail[label] = {'status': status, 'attempt': view} + return checks, detail + + +def scenario_io(bench, gateway, tty): + if tty: + started = gateway.exec('gov', ['sh', '-c', 'stty size; read line; stty size; echo "got-$line"'], tty=True) + checks = [started_checks('a reading shell', started)] + if started['outcome'] != 'started': + return checks, {'started': started} + pid = started['process_id'] + first, cursor, _, found = gateway.read_until(pid, '24 80') + resized, _ = gateway.call('process_resize', {'process_id': pid, 'rows': 40, 'cols': 100}) + gateway.call('write_stdin', {'process_id': pid, 'data': 'x\n'}) + second, cursor, _, echoed = gateway.read_until(pid, 'got-x', cursor=cursor) + status = gateway.wait_exited(pid) + checks += [check('the terminal starts at 24x80', found, output=first), + check('process_resize answers 40x100', resized.get('rows') == 40 and resized.get('cols') == 100, + answer=resized), + check('stdin reaches the process', echoed, output=second), + check('the process sees the new size', '40 100' in second, output=second), + check('it exits 0', (status or {}).get('termination') == 'exited' and status.get('exit_code') == 0, + status=status)] + else: + started = gateway.exec('gov', ['cat']) + checks = [started_checks('cat', started)] + if started['outcome'] != 'started': + return checks, {'started': started} + pid = started['process_id'] + gateway.call('write_stdin', {'process_id': pid, 'data': 'ping-1\n'}) + text, _, _, echoed = gateway.read_until(pid, 'ping-1') + try: + gateway.call('process_resize', {'process_id': pid, 'rows': 40, 'cols': 100}) + resize = None + except ToolError as error: + resize = error.code + gateway.terminate(pid) + status = gateway.wait_exited(pid) + checks += [check('stdin reaches the process and its output returns', echoed, output=text), + check('process_resize refuses a pipe', resize == 'PROCESS_NOT_TTY', code=resize), + check('terminated', (status or {}).get('termination') == 'terminated', status=status)] + account, view = account_checks(bench, gateway, pid, MATRIX_REQUEST) + return checks + account, {'attempt': view} + + +def scenario_termination(bench, gateway, tty): + started = gateway.exec('gov', ['sleep', '600'], tty=tty) + checks = [started_checks('sleep', started)] + if started['outcome'] != 'started': + return checks, {'started': started} + pid = started['process_id'] + root = (gateway.execution(pid) or {}).get('scope_root_pid') + asked = time.monotonic() + ack = gateway.terminate(pid) + status = gateway.wait_exited(pid) + exited_ms = round((time.monotonic() - asked) * 1000, 3) + scope = host.wait_group_gone(root) if root else None + checks += [check('terminate_process answered', ack is not None, ack_ms=ack), + check('reported terminated', (status or {}).get('termination') == 'terminated', status=status), + check("the scope's process group ended", scope is not None, root=root, + scope_exit_ms=None if scope is None else round(scope * 1000 + exited_ms, 3))] + account, view = account_checks(bench, gateway, pid, MATRIX_REQUEST) + return checks + account, {'ack_ms': ack, 'exited_ms': exited_ms, 'attempt': view} + + +def scenario_timeout(bench, gateway, tty): + started = gateway.exec('gov', ['sleep', '600'], tty=tty) + checks = [started_checks('sleep', started)] + if started['outcome'] != 'started': + return checks, {'started': started} + status = gateway.wait_exited(started['process_id'], limit=MATRIX_TIMEOUT_S + 45) + checks.append(check(f'timed out after {MATRIX_TIMEOUT_S} s', (status or {}).get('termination') == 'timeout', + status=status)) + account, view = account_checks(bench, gateway, started['process_id'], MATRIX_REQUEST) + return checks + account, {'attempt': view} + + +def scenario_exec_failed(bench, gateway, tty): + """READY, then the program cannot be executed: an interpreter that does not exist (ENOENT, + helper exit 127) or one that is not executable (EACCES, helper exit 126).""" + root = bench.roots['gov'] + plain = root / 'plain-file' + plain.write_text('not a program\n') + os.chmod(plain, 0o644) + scripts = {'ENOENT': (root / 'missing-interpreter', '#!/nonexistent/interpreter\n', 2), + 'EACCES': (root / 'unusable-interpreter', f'#!{plain}\n', 13)} + checks, detail = [], {} + for label, (path, text, errno) in scripts.items(): + path.write_text(text) + os.chmod(path, 0o755) + result = gateway.exec('gov', [str(path)], tty=tty) + checks += prefixed(label, refusal_checks(bench, result, 'PROCESS_SPAWN_FAILED', 'released')) + checks.append(check(f'{label}: the refusal gives errno {errno}', f'(errno {errno})' in (result.get('message') or ''), + message=result.get('message'))) + attempt = result.get('attempt_id') + view = host.journal_attempts([attempt], bench.database, bench.authority['consumer']).get(attempt) if attempt else None + checks.append(check(f'{label}: the helper claimed the grant and its scope settled it', + bool(view and view['launched'] and view['release_reason'] == 'scope_terminated'), view=view)) + detail[label] = {'result': result, 'attempt': view} + return checks, detail + + +def scenario_shortage(bench, gateway, tty): + """A request larger than the work capacity is denied every time, and each refusal frees the + workspace: the next call is refused for capacity again, never WORKSPACE_BUSY.""" + results = [gateway.exec('hog', ['true'], tty=tty) for _ in range(3)] + checks = [] + for index, result in enumerate(results, 1): + checks += prefixed(f'call {index}', refusal_checks(bench, result, 'RESOURCE_UNAVAILABLE', 'denied', + 'resource_unavailable')) + attempts = [result.get('attempt_id') for result in results] + checks.append(check('each call is a new attempt', len(set(attempts)) == len(attempts) and None not in attempts, + attempts=attempts)) + return checks, {'latency_ms': [result.get('latency_ms') for result in results]} + + +def scenario_unsupported(bench, gateway, tty): + result = gateway.exec('kernel', ['true'], tty=tty) + return refusal_checks(bench, result, 'RESOURCE_POLICY_UNSUPPORTED', 'denied', 'resource_policy_unsupported'), { + 'result': result} + + +def child_running(owner, needle, limit=10.0): + """The PID of the owner's child whose command line is `needle`.""" + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + for pid in host.children(owner): + if host.run_text('ps', '-o', 'command=', '-p', str(pid)) == needle: + return pid + time.sleep(0.1) + return None + + +def scenario_mixed(bench, gateway, tty): + """A governed and an ungoverned execution of one gateway at once: each runs, and neither holds + a descriptor beyond its standard three, so the launch's permit carrier and transcript reached + the helper only.""" + off = gateway.exec('off', ['sleep', '31'], tty=tty) + gov = gateway.exec('gov', ['sleep', '32'], tty=tty) + checks = [check('the ungoverned execution started', off['outcome'] == 'started', started=off), + started_checks('the governed execution', gov)] + detail = {} + if off['outcome'] == 'started' and gov['outcome'] == 'started': + owner = bench.owner_pid or gateway.pid + governed = (gateway.execution(gov['process_id']) or {}).get('scope_root_pid') + ungoverned = child_running(owner, 'sleep 31') + detail = {'governed_pid': governed, 'ungoverned_pid': ungoverned, + 'governed_descriptors': host.descriptors(governed) if governed else None, + 'ungoverned_descriptors': host.descriptors(ungoverned) if ungoverned else None} + checks += [check('the ungoverned child was found', ungoverned is not None, owner=owner), + check('the governed child holds only its standard descriptors', + detail['governed_descriptors'] == [0, 1, 2], descriptors=detail['governed_descriptors']), + check('the ungoverned child holds only its standard descriptors', + detail['ungoverned_descriptors'] == [0, 1, 2], descriptors=detail['ungoverned_descriptors'])] + for started in (off, gov): + if started['outcome'] == 'started': + gateway.terminate(started['process_id']) + gateway.wait_exited(started['process_id']) + if gov['outcome'] == 'started': + account, view = account_checks(bench, gateway, gov['process_id'], MATRIX_REQUEST) + checks += account + detail['attempt'] = view + return checks, detail + + +def scenario_no_helper(bench, helper, code_label): + """Rollback of new admissions: a gateway without a usable launch helper admits, then cancels, + and still runs ungoverned workspaces.""" + gateway = bench.gateway(code_label, helper=helper) + try: + result = gateway.exec('gov', ['true']) + checks = refusal_checks(bench, result, 'MANAGED_LAUNCH_UNAVAILABLE', 'cancelled') + checks += off_still_runs(gateway) + finally: + bench.close(gateway) + return checks, {'result': result} + + +def off_still_runs(gateway): + started = gateway.exec('off', ['true']) + status = gateway.wait_exited(started['process_id']) if started['outcome'] == 'started' else None + return [check('an off workspace still runs', (status or {}).get('exit_code') == 0, started=started, status=status)] + + +def scenario_status_mode(bench): + gateway = bench.gateway('status-mode', mode='status', helper=None) + try: + result = gateway.exec('gov', ['true']) + checks = refusal_checks(bench, result, 'RESOURCE_POLICY_UNSUPPORTED', None) + off_still_runs(gateway) + finally: + bench.close(gateway) + return checks, {'result': result} + + +def scenario_authority_unavailable(bench): + """No authority at the socket: refused promptly, nothing reaches DevGuard.""" + gateway = bench.gateway('no-authority', socket=bench.work / 'missing' / 'authority.sock') + try: + result = gateway.exec('gov', ['true']) + checks = refusal_checks(bench, result, 'RESOURCE_AUTHORITY_UNAVAILABLE', 'absent') + off_still_runs(gateway) + finally: + bench.close(gateway) + return checks, {'result': result} + + +def scenario_runner_socket(bench): + """`uds` with a worker the gateway did not start: governed executions are unsupported.""" + socket = bench.work / 'standalone.sock' + worker = subprocess.Popen([str(bench.args.runtime_bin), str(socket)], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=open(bench.evidence.file(f'{bench.mode}/standalone-worker.log'), 'ab'), + env=host.minimal_env(bench.work), start_new_session=True) + bench.live.workers.append(worker) + deadline = time.monotonic() + 15 + while not socket.exists() and time.monotonic() < deadline and worker.poll() is None: + time.sleep(0.05) + gateway = bench.gateway('runner-socket', runner_socket=socket) + try: + result = gateway.exec('gov', ['true']) + checks = refusal_checks(bench, result, 'RESOURCE_POLICY_UNSUPPORTED', None) + off_still_runs(gateway) + finally: + bench.close(gateway) + worker.terminate() + worker.wait(timeout=10) + return checks, {'result': result} + + +def matrix_workspaces(work, capacity): + """`gov` and `gov2` require participation; `off` does not; `kernel` asks for kernel task + control, which macOS cannot give; `hog` asks for more CPU than the whole work capacity.""" + hog = dict(MATRIX_REQUEST, cpu_milli=((capacity or {}).get('cpu_milli') or 1_000_000) + 1) + governed = host.required(**MATRIX_REQUEST) + return {'gov': (work / 'ws/gov', governed), 'gov2': (work / 'ws/gov2', governed), 'off': (work / 'ws/off', None), + 'kernel': (work / 'ws/kernel', host.required(**MATRIX_REQUEST, minimum=KERNEL_MINIMUM)), + 'hog': (work / 'ws/hog', host.required(**hog))} + + +def settled_checks(bench, label): + """After a mode: nothing of the consumer is charged, and its instances are not active.""" + deadline = time.monotonic() + 60 + while True: + charged = host.journal_charged(bench.database) + mine = charged['attempts'].get(bench.authority['consumer'], 0) + if not mine or time.monotonic() >= deadline: + break + time.sleep(1) + instances = host.journal_instances(bench.authority['consumer'], bench.database) + return [check(f'{label}: nothing of the consumer stays charged', mine == 0, charged=charged), + check(f'{label}: no instance stays active', not any(entry['state'] == 'active' for entry in instances), + instances=instances)] + + +def installed_authority(args): + return {'socket': args.socket, 'credential_file': args.credential_file, 'helper': host.Release(args.release).helper, + 'consumer': host.CONSUMER, 'generation': host.GENERATION, 'journal': args.journal} + + +def matrix(args, live, evidence): + release = host.Release(args.release) + before = {'state': release.state(), 'charged': host.journal_charged(args.journal), + 'pressure': host.pressure_state(args.service_log), 'work_capacity': host.work_capacity(args.service_log)} + evidence.write_json('before.json', before) + if not before['state']['runs_release']: + return {'verdict': 'incomplete', 'problems': ['the service does not run the release'], 'scenarios': []} + if sum(before['charged']['attempts'].values()) or before['charged']['leases']: + return {'verdict': 'incomplete', 'problems': ['work was charged when the matrix began'], 'scenarios': []} + log_offset = host.log_size(args.service_log) + writer = host.Writer() + results = [] + authority = installed_authority(args) + try: + for mode in args.modes: + work = scratch() + bench = None + try: + bench = Bench(args, live, evidence, writer, mode, work, + matrix_workspaces(work, before['work_capacity']), authority) + gateway = bench.gateway('main', timeout_s=MATRIX_TIMEOUT_S) + entry = run_scenario(results, evidence, mode, None, 'registration', scenario_registration, bench, gateway) + if entry['verdict'] == 'pass': + for tty in IO: + for name, function in (('exits', scenario_exits), ('io', scenario_io), + ('termination', scenario_termination), ('timeout', scenario_timeout), + ('exec-failed', scenario_exec_failed), ('shortage', scenario_shortage), + ('unsupported-minimum', scenario_unsupported), ('mixed', scenario_mixed)): + run_scenario(results, evidence, mode, io_name(tty), name, function, bench, gateway, tty) + bench.close(gateway) + run_scenario(results, evidence, mode, None, 'no-helper', scenario_no_helper, bench, None, 'no-helper') + run_scenario(results, evidence, mode, None, 'unusable-helper', scenario_no_helper, bench, + Path('/nonexistent/devguard-launch'), 'unusable-helper') + run_scenario(results, evidence, mode, None, 'status-mode', scenario_status_mode, bench) + run_scenario(results, evidence, mode, None, 'authority-unavailable', scenario_authority_unavailable, bench) + if mode == 'uds': + run_scenario(results, evidence, mode, None, 'runner-socket', scenario_runner_socket, bench) + entry = run_scenario(results, evidence, mode, None, 'settled', lambda: (settled_checks(bench, mode), {})) + if entry['verdict'] != 'pass': + print('qualify: work stayed charged; the matrix stops here', flush=True) + break + finally: + if bench is not None: + bench.teardown() + else: + shutil.rmtree(work, ignore_errors=True) + finally: + evidence.write_json('writer.json', writer.close()) + evidence.write_json('service-events.json', host.service_events(args.service_log, log_offset or 0)) + evidence.write_json('after.json', {'state': release.state(), 'charged': host.journal_charged(args.journal), + 'instances': host.journal_instances(database=args.journal)}) + verdict = analysis.suite_verdict([entry['verdict'] for entry in results]) + return {'verdict': verdict, 'scenarios': [{key: entry[key] for key in ('scenario', 'mode', 'io', 'verdict')} + for entry in results]} + + +# ---------------------------------------------------------------- faults (DevGuard's fixture authority) +# Each case gets its own fixture authority (DevGuard's daemon code at CodeSpace's pin, in a private +# state directory), because it leaves Suspect or charged work behind, kills the authority or holds +# its lock. The expectations are DevGuard's contracts ("Native policy application and scope +# evidence", "Reconciliation") as CodeSpace's gateway tests of the same cases assert them. + +# What a fault case's execution asks for, as CodeSpace's gateway tests ask. +FIXTURE_REQUEST = {'cpu_milli': 50, 'memory_bytes': 32 << 20, 'tasks': 4} +# Process control after a lost authority: the reaper's two-second bound for an observation that +# cannot be answered, twice. +LOSS_LIMIT_S = 4.0 +# How long the stalled-authority case holds the authority's lock. +HOLD_MS = 1500 + + +def fault_workspaces(work): + """`gov-N` require participation, `gov-confirm` also an approval; `off` does not.""" + governed = host.required(**FIXTURE_REQUEST) + spaces = {'off': (work / 'ws/off', None), + 'gov-confirm': (work / 'ws/gov-confirm', governed, {'approvals': 'confirm'})} + for n in range(3): + spaces[f'gov-{n}'] = (work / f'ws/gov-{n}', governed) + return spaces + + +def fixture_bench(args, live, evidence, writer, mode, name, workspaces=fault_workspaces): + """A fresh fixture authority and a bench of gateways against it.""" + work = scratch() + try: + fixture = host.FixtureAuthority(args.fixture_bin, work / 'fx', host.minimal_env(work), + evidence.file(f'{mode}/{name}/fixture.log')) + except Exception: + shutil.rmtree(work, ignore_errors=True) + raise + live.fixtures.append(fixture) + settings = fixture.settings + if not settings.get('native'): + fixture.stop() + shutil.rmtree(work, ignore_errors=True) + raise NotNative('the fixture authority has no native host evidence on this platform') + authority = {'socket': settings['socket'], 'credential_file': settings['credential_file'], + 'helper': args.fixture_helper, 'consumer': settings['consumer'], + 'generation': settings['generation'], 'journal': fixture.journal()} + return Bench(args, live, evidence, writer, mode, work, workspaces(work), authority, prefix=f'{mode}/{name}', + fixture=fixture) + + +class NotNative(Exception): + """The fixture authority cannot govern on this platform (Linux before DG-LINUX).""" + + +def run_fault(results, args, live, evidence, writer, mode, io, name, function): + """One fault case on its own fixture authority, torn down however the case ends.""" + label = name if io is None else f'{name}-{io}' + holder = {} + + def case(): + bench = holder['bench'] = fixture_bench(args, live, evidence, writer, mode, label) + return function(bench) if io is None else function(bench, io == 'pty') + try: + return run_scenario(results, evidence, mode, io, name, case) + finally: + if 'bench' in holder: + holder['bench'].teardown() + + +def view_of(bench, attempt): + """DevGuard's record of one attempt now, or None.""" + if not attempt: + return None + return host.journal_attempts([attempt], bench.database, bench.authority['consumer']).get(attempt) + + +def settled_view(bench, attempt, limit=30.0): + """DevGuard's record of one attempt once it holds nothing, or as it stands after `limit`.""" + if not attempt: + return None + found, _ = host.settle_attempts([attempt], bench.database, limit, bench.authority['consumer']) + return found.get(attempt) + + +def released_cleanly(view): + return bool(view and view['phase'] == 'released' and view['release_reason'] == 'scope_terminated' + and not view['charged'] and view['tracking_lost'] is False) + + +def suspect(view): + return bool(view and view['phase'] == 'suspect' and view['charged']) + + +def read_all(gateway, process_id, limit=10.0): + return gateway.read_until(process_id, None, limit=limit)[0] + + +def until(moment): + time.sleep(max(0.0, moment - time.monotonic())) + + +def attempt_of(gateway, process_id): + return (gateway.execution(process_id) or {}).get('attempt_id') + + +def ran(gateway, started, output, label='it'): + """The checks of a governed command that should have run to exit 0 with `output`.""" + if started['outcome'] != 'started': + return [started_checks(label, started)], None, None + status = gateway.wait_exited(started['process_id']) + text = read_all(gateway, started['process_id']) + return [started_checks(label, started), + check(f'{label} printed {output!r} and exited 0', text == output and (status or {}).get('exit_code') == 0, + output=text, status=status)], status, text + + +def fault_descendant(bench, tty): + """A root that exits leaving a descendant: the attempt stays charged, not Suspect, while the + descendant runs, and is released once it ends. On a PTY the reconciler is paused around the + exit, so only the owner's observation before the reap can adopt the descendant.""" + gateway = bench.gateway('main') + script = ("trap '' HUP; " if tty else '') + 'sleep 3 /dev/null 2>&1 & echo started' + if tty: + bench.fixture.command('pause') + try: + started = gateway.exec('gov-0', ['/bin/sh', '-c', script], tty=tty) + checks, status, _ = ran(gateway, started, 'started\n', 'the root') + finally: + if tty: + bench.fixture.command('resume') + attempt = attempt_of(gateway, started.get('process_id')) + time.sleep(1.5) + during = view_of(bench, attempt) + after = settled_view(bench, attempt) + checks += [check('the descendant keeps the attempt charged, not Suspect', + bool(during and during['charged'] and during['phase'] != 'suspect'), view=during), + check('released once the descendant ended, tracking complete', released_cleanly(after), view=after)] + return checks, {'status': status, 'during': during, 'after': after} + + +def fault_escape(bench, tty): + """A descendant that leaves the scope's process group with setsid while the root still runs: + DevGuard finds a child of a live member outside the group, an escape, which is sticky and + keeps the attempt charged as Suspect after everything has ended. The descendant is this + harness's own Python, so the host needs no setsid program.""" + gateway = bench.gateway('main') + script = ('"$0" -c "import os, time; os.setsid(); time.sleep(3)" /dev/null 2>&1 & ' + 'sleep 2; echo started') + began = time.monotonic() + started = gateway.exec('gov-0', ['/bin/sh', '-c', script, sys.executable], tty=tty) + checks, status, _ = ran(gateway, started, 'started\n', 'the root') + attempt = attempt_of(gateway, started.get('process_id')) + until(began + 7.0) + view = view_of(bench, attempt) + checks.append(check('the escape keeps the attempt charged as Suspect after everything ended', suspect(view), + view=view)) + return checks, {'status': status, 'attempt': view} + + +def fault_stalled_observer(bench, tty): + """The owner's observation before the reap cannot be answered: the reconciler is paused and + the authority held from just before the root exits, leaving a descendant. CodeSpace reaps the + root at its observer's bound while the authority is still held, so process control never + waits on DevGuard; the unproven descendant keeps the attempt charged as Suspect, also after + it ends (CSRG-U5).""" + gateway = bench.gateway('main') + go = bench.roots['gov-0'] / 'go' + script = (f"while [ ! -e '{go}' ]; do sleep 0.05; done; " + "trap '' HUP; sleep 6 /dev/null 2>&1 & echo started") + started = gateway.exec('gov-0', ['/bin/sh', '-c', script], tty=tty) + checks = [started_checks('the root', started)] + if started['outcome'] != 'started': + return checks, {'started': started} + pid = started['process_id'] + attempt = attempt_of(gateway, pid) + bench.fixture.command('pause') + held = time.monotonic() + bench.fixture.command('hold 4000') + go.touch() + status = gateway.wait_exited(pid, limit=10) + reaped_s = time.monotonic() - held + output = read_all(gateway, pid) + until(held + 4.5) + bench.fixture.command('resume') + time.sleep(1.5) + middle = view_of(bench, attempt) + until(held + 8.0) + end = view_of(bench, attempt) + checks += [check('the root printed and exited 0', output == 'started\n' and (status or {}).get('exit_code') == 0, + output=output, status=status), + check('reaped at the observer bound while the authority was still held', + status is not None and reaped_s < 3.9, reaped_s=round(reaped_s, 3)), + check('the unproven descendant leaves the attempt Suspect', suspect(middle), view=middle), + check('still Suspect after the descendant ended', suspect(end), view=end)] + return checks, {'reaped_s': reaped_s, 'middle': middle, 'end': end} + + +def fault_authority_lost(bench): + """The authority dies while a governed pipe and PTY process run: CodeSpace keeps answering for + both and terminates them as any other process, within the reaper's bound for an observation + that cannot be answered; their workspaces are free again, where a new governed execution is + refused having started nothing; ungoverned ones still run.""" + gateway = bench.gateway('main') + checks, running, detail = [], [], {} + for workspace, tty in (('gov-0', False), ('gov-1', True)): + started = gateway.exec(workspace, ['/bin/sleep', '30'], tty=tty) + checks.append(started_checks(f'{workspace} ({io_name(tty)})', started)) + if started['outcome'] == 'started': + running.append((workspace, started['process_id'])) + bench.fixture.kill() + began = time.monotonic() + for workspace, pid in running: + payload, latency = gateway.status(pid) + detail[workspace] = {'state': payload.get('state'), 'status_ms': latency, 'ack_ms': gateway.terminate(pid)} + for workspace, pid in running: + status = gateway.wait_exited(pid, limit=10) + detail[workspace]['status'] = status + checks.append(check(f'{workspace}: answered running, then terminated', + detail[workspace]['state'] == 'running' and (status or {}).get('termination') == 'terminated', + **detail[workspace])) + elapsed = time.monotonic() - began + checks.append(check(f'both ended within {LOSS_LIMIT_S} s of the loss', len(running) == 2 and elapsed < LOSS_LIMIT_S, + elapsed_s=round(elapsed, 3))) + for workspace, _ in running: + result = gateway.exec(workspace, ['/usr/bin/true']) + detail[workspace]['afterwards'] = result + checks += prefixed(f'{workspace} afterwards', + refusal_checks(bench, result, 'RESOURCE_AUTHORITY_UNAVAILABLE', 'absent')) + return checks + off_still_runs(gateway), detail + + +def fault_authority_restart(bench): + """The authority restarts while a governed process runs, one held for approval and resumed: + CodeSpace keeps the process and its status; once it ends, DevGuard, which no longer knows its + scope, keeps the attempt charged as Suspect; resuming the approval again returns the same + process, never a second launch; a new execution is admitted and released (CSRG-U5).""" + gateway = bench.gateway('main') + go = bench.roots['gov-confirm'] / 'go' + held = gateway.exec('gov-confirm', ['/bin/sh', '-c', f"while [ ! -e '{go}' ]; do sleep 0.05; done; echo done"]) + approval = held.get('approval_id') + checks = [check('held for approval', held['outcome'] == 'refused' and held.get('code') == 'APPROVAL_REQUIRED' + and approval is not None, held=held)] + if not approval: + return checks, {'held': held} + gateway.call('approval_resolve', {'approval_id': approval, 'decision': 'grant'}) + resumed, _ = gateway.resume(approval) + pid = (resumed.get('exec_command') or {}).get('process_id') + checks.append(check('resumed and confirmed', (resumed.get('exec_command') or {}).get('dispatch_status') == 'confirmed', + resumed=resumed)) + if not pid: + return checks, {'resumed': resumed} + attempt = attempt_of(gateway, pid) + restarted = bench.fixture.command('restart') + payload, latency = gateway.status(pid) + go.touch() + status = gateway.wait_exited(pid) + output = read_all(gateway, pid) + time.sleep(1.5) + after = view_of(bench, attempt) + again, _ = gateway.resume(approval) + fresh = gateway.exec('gov-0', ['/bin/echo', 'after']) + fresh_checks, _, _ = ran(gateway, fresh, 'after\n', 'a new execution') + fresh_view = settled_view(bench, attempt_of(gateway, fresh.get('process_id'))) + still = view_of(bench, attempt) + checks += [check('the authority restarted', restarted == {'restarted': True}, answer=restarted), + check('the process still runs under CodeSpace', payload.get('state') == 'running', status_ms=latency), + check('it ran to its end', output == 'done\n' and (status or {}).get('exit_code') == 0, + output=output, status=status), + check('its attempt stays charged as Suspect', suspect(after), view=after), + check('resuming again returns the same process', + (again.get('exec_command') or {}).get('process_id') == pid, again=again), + check('the log reports one launch of it', gateway.launches().get(pid) == 1, + launches=gateway.launches().get(pid))] + checks += fresh_checks + checks += [check('the new execution was released', released_cleanly(fresh_view), view=fresh_view), + check('the Suspect attempt is still charged', suspect(still), view=still)] + return checks, {'after': after, 'fresh': fresh_view, 'still': still} + + +def fault_authority_stalled(bench): + """The authority holds its lock past the owner's deadlines for the whole exec_command, so the + session's first request that needs it gets no answer in time (registration, or the admission + once registered). Whichever it was, the call is refused before the hold ends, the command + never runs, DevGuard never hands the attempt to a helper and nothing of it stays charged once + an admission it took late has expired; the workspace's next execution is a new attempt and + runs. A lost answer to a request already past registration is not reachable from outside the + owner; DevGuard's client and the runner test it (docs/cs-rg-qualification.md).""" + gateway = bench.gateway('main') + marker = bench.roots['gov-0'] / 'ran' + bench.fixture.command(f'hold {HOLD_MS}') + held = time.monotonic() + result = gateway.exec('gov-0', ['/bin/sh', '-c', f"touch '{marker}'"]) + answered_s = time.monotonic() - held + # A prepared admission DevGuard took after the hold expires on its five-second deadline. + until(held + HOLD_MS / 1000 + 6.0) + attempt = result.get('attempt_id') + view = settled_view(bench, attempt) + following = gateway.exec('gov-0', ['/bin/echo', 'next']) + next_checks, _, _ = ran(gateway, following, 'next\n', 'the next execution') + checks = [check('refused as unknown or unavailable', result['outcome'] == 'refused' + and result.get('code') in ('ADMISSION_UNKNOWN', 'RESOURCE_AUTHORITY_UNAVAILABLE'), result=result), + check('answered before the hold ended', answered_s < HOLD_MS / 1000, answered_s=round(answered_s, 3)), + check('the command never ran', not marker.exists()), + check('DevGuard never handed it to a helper', view is None or view['scope_root_pid'] is None, view=view), + check('nothing of it stays charged', not (view and view['charged']), view=view)] + checks += next_checks + checks.append(check('the next execution is a new attempt', + attempt_of(gateway, following.get('process_id')) not in (None, attempt))) + return checks, {'result': result, 'attempt': view} + + +def fault_helper_ends(bench): + """A launch helper that exits before it reports READY (a stand-in that exits 1): the command is + refused as never attempted, and once CodeSpace has reaped the helper it reports that it holds + none, so DevGuard releases the unclaimed grant as never started.""" + fake = bench.work / 'ending-helper' + fake.write_text('#!/bin/sh\nexit 1\n') + os.chmod(fake, 0o755) + gateway = bench.gateway('ending-helper', helper=fake) + marker = bench.roots['gov-0'] / 'ran' + result = gateway.exec('gov-0', ['/bin/sh', '-c', f"touch '{marker}'"]) + view = settled_view(bench, result.get('attempt_id')) + checks = refusal_checks(bench, result, 'PROCESS_SPAWN_FAILED', 'released') + checks += [check('the helper ended before READY', 'ended before it reported ready' in (result.get('message') or ''), + message=result.get('message')), + check('the command never ran', not marker.exists()), + check('released as never started, never claimed', + bool(view and view['release_reason'] == 'no_helper_created' and view['scope_root_pid'] is None + and not view['launched']), view=view)] + return checks, {'result': result, 'attempt': view} + + +def fault_credential(bench): + """A credential the authority refuses (a fresh random secret) and one that cannot be read (a + missing file): every governed execution is refused before anything is asked, and ungoverned + ones still run.""" + wrong = bench.work / 'wrong.secret' + descriptor = os.open(wrong, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, 'w') as handle: + handle.write(secrets.token_hex(32)) + checks, detail = [], {} + for label, path, state in (('refused credential', wrong, 'credential_refused'), + ('missing credential', bench.work / 'missing.secret', 'credential_unavailable')): + gateway = bench.gateway(label.replace(' ', '-'), credential_file=path) + try: + result = gateway.exec('gov-0', ['/usr/bin/true']) + detail[label] = result + checks += prefixed(label, refusal_checks(bench, result, 'RESOURCE_AUTHORITY_UNAVAILABLE', 'absent')) + checks.append(check(f'{label}: the refusal names the registration state {state}', + f'(registration: {state}' in (result.get('message') or ''), message=result.get('message'))) + checks += prefixed(label, off_still_runs(gateway)) + finally: + bench.close(gateway) + return checks, detail + + +def fault_race(bench): + """terminate_process around a governed process's own exit, alternating pipe and PTY: each ends + once with one final status, is launched once and released, and nothing stays charged.""" + gateway = bench.gateway('main') + rows, attempts = [], [] + for index in range(12): + tty = bool(index % 2) + started = gateway.exec('gov-0', ['/bin/sleep', '0.3'], tty=tty) + if started['outcome'] != 'started': + rows.append({'index': index, 'started': started}) + continue + time.sleep(0.25 + 0.01 * (index % 6)) + try: + gateway.terminate(started['process_id']) + answer = 'acknowledged' + except ToolError as error: + answer = 'transport' if error.transport else error.code + status = gateway.wait_exited(started['process_id']) + attempts.append(attempt_of(gateway, started['process_id'])) + rows.append({'index': index, 'tty': tty, 'terminate': answer, 'termination': (status or {}).get('termination')}) + found, charged = host.settle_attempts([a for a in attempts if a], bench.database, 30, bench.authority['consumer']) + launches = gateway.launches() + checks = [check('every round started', all('started' not in row for row in rows), rows=rows), + check('terminate_process always answered', all(row.get('terminate') != 'transport' for row in rows)), + check('each ended once as exited or terminated', + all(row.get('termination') in ('exited', 'terminated') for row in rows if 'tty' in row)), + check('each was launched once', len(attempts) == len(rows) and all(count == 1 for count in launches.values()), + launches=launches), + check('each was released, tracking complete', + None not in attempts and all(released_cleanly(found.get(a)) for a in attempts), charged=charged)] + return checks, {'rounds': rows} + + +def fault_shutdown(bench): + """The gateway shuts down, its input closed, while a governed pipe and PTY process run (in + `uds` mode the worker goes with it): their scopes end, DevGuard releases them once it sees + that, and a new gateway registers and runs without replaying either.""" + gateway = bench.gateway('main') + authority, _ = bench.registration(gateway, 'gov-0') + owner = bench.owner_pid + checks, roots, attempts = [], [], [] + for workspace, tty in (('gov-0', False), ('gov-1', True)): + started = gateway.exec(workspace, ['/bin/sleep', '30'], tty=tty) + checks.append(started_checks(f'{workspace} ({io_name(tty)})', started)) + entry = gateway.execution(started.get('process_id')) or {} + roots.append(entry.get('scope_root_pid')) + attempts.append(entry.get('attempt_id')) + closed = gateway.close() + ended = [host.wait_group_gone(root, limit=15) if root else None for root in roots] + found, charged = host.settle_attempts([a for a in attempts if a], bench.database, 30, bench.authority['consumer']) + successor = bench.gateway('successor') + registered, _ = bench.registration(successor, 'gov-0') + fresh = successor.exec('gov-0', ['/bin/echo', 'fresh']) + fresh_checks, _, _ = ran(successor, fresh, 'fresh\n', 'the successor') + launched = [view for view in host.journal_attempts(None, bench.database, bench.authority['consumer']).values() + if view['launched']] + checks += [check('the gateway exited by itself', closed['stopped'] is None and closed['returncode'] == 0, **closed), + check('the execution owner is gone', gone(owner) if owner else False, owner=owner), + check("both scopes' process groups ended", None not in roots and None not in ended, + ended_s=[None if value is None else round(value, 3) for value in ended]), + check('both attempts were released, tracking complete', + None not in attempts and all(released_cleanly(found.get(a)) for a in attempts), charged=charged, + views=[found.get(a) for a in attempts]), + check('the successor registered', (registered.get('registration') or {}).get('state') == 'registered', + authority=registered)] + checks += fresh_checks + checks.append(check('nothing was launched again', len(launched) == 3, launched=len(launched))) + return checks, {'closed': closed, 'authority': authority} + + +def fault_worker_killed(bench): + """`uds`: the worker is killed while it prepares a governed execution (the authority held, so + its session waits): the gateway answers the call, the command never runs, DevGuard never + hands the attempt to a helper and holds nothing of it once a late admission expired.""" + gateway = bench.gateway('main') + bench.registration(gateway, 'gov-0') + worker = bench.owner_pid + marker = bench.roots['gov-0'] / 'ran' + answer = {} + + def call(): + answer.update(gateway.exec('gov-0', ['/bin/sh', '-c', f"touch '{marker}'"])) + bench.fixture.command(f'hold {HOLD_MS}') + held = time.monotonic() + caller = threading.Thread(target=call, daemon=True) + caller.start() + time.sleep(0.1) + killed = False + if worker and worker != gateway.pid: + try: + os.kill(worker, signal.SIGKILL) + killed = True + except ProcessLookupError: + pass + caller.join(timeout=60) + until(held + HOLD_MS / 1000 + 6.0) + view = settled_view(bench, answer.get('attempt_id')) + alive = gateway.process.poll() is None + try: + later = gateway.exec('gov-1', ['/bin/echo', 'later']) + except ToolError as error: + later = {'outcome': 'error', 'code': error.code} + checks = [check('the worker was killed during the call', killed and caller.is_alive() is False, worker=worker), + check('the gateway answered the call', answer.get('outcome') in ('refused', 'started') + and answer.get('code') not in host.TRANSPORT_CODES, answer=answer), + check('the command never ran', not marker.exists()), + check('DevGuard never handed it to a helper', view is None or view['scope_root_pid'] is None, view=view), + check('nothing of it stays charged', not (view and view['charged']), view=view), + check('the gateway still runs', alive)] + return checks, {'answer': answer, 'attempt': view, 'later': later} + + +FAULTS = [('descendant', IO, fault_descendant), ('escape', IO, fault_escape), + ('stalled-observer', IO, fault_stalled_observer), ('authority-lost', None, fault_authority_lost), + ('authority-restart', None, fault_authority_restart), ('authority-stalled', None, fault_authority_stalled), + ('helper-ends', None, fault_helper_ends), ('credential', None, fault_credential), + ('terminate-race', None, fault_race), ('shutdown', None, fault_shutdown)] + + +def faults(args, live, evidence): + if platform.system() != 'Darwin': + return {'verdict': 'incomplete', 'problems': ['the fault cases need native host evidence (macOS)'], + 'scenarios': []} + writer = host.Writer() + results = [] + try: + for mode in args.modes: + cases = FAULTS + ([('worker-killed', None, fault_worker_killed)] if mode == 'uds' else []) + for name, ios, function in cases: + for tty in (ios if ios else (None,)): + run_fault(results, args, live, evidence, writer, mode, None if tty is None else io_name(tty), + name, function) + finally: + evidence.write_json('writer.json', writer.close()) + verdict = analysis.suite_verdict([entry['verdict'] for entry in results]) + return {'verdict': verdict, 'scenarios': [{key: entry[key] for key in ('scenario', 'mode', 'io', 'verdict')} + for entry in results]} + + +# ---------------------------------------------------------------- control-path sampling + +class Schedule(threading.Thread): + """Runs `action(slot)` on a fixed schedule from `start` (monotonic seconds): every `period` + seconds, or at the given `offsets`, until stopped. A slot whose moment passed while the + previous action still ran is recorded as missed, never run late, as DevGuard's protocol + counts it; so is one that passed before `stop` while the last action ran. Every row goes to + the writer and is kept.""" + + def __init__(self, writer, path, action, start, period=None, offsets=None): + super().__init__(daemon=True) + self.writer, self.path, self.action, self.begin = writer, path, action, start + self.period, self.offsets = period, offsets + self.halt = threading.Event() + self.rows = [] + self.ended = start + self.stopped_at = None + + def moments(self): + if self.offsets is not None: + return (self.begin + offset for offset in self.offsets) + return (self.begin + slot * self.period for slot in itertools.count()) + + def keep(self, row): + self.rows.append(row) + self.writer.write(self.path, row) + + def run(self): + for slot, moment in enumerate(self.moments()): + if self.halt.is_set() and (self.stopped_at is None or moment >= self.stopped_at): + return + if self.ended > moment: + self.keep({'slot': slot, 'unix': time.time(), 'outcome': 'missed_slot'}) + continue + if self.halt.wait(max(0.0, moment - time.monotonic())): + return + row = {'slot': slot, 'unix': time.time()} + try: + row.update(self.action(slot)) + except ToolError as error: + row.update(outcome='error', code=error.code, message=error.message[:300], latency_ms=error.latency_ms) + except Exception as error: # noqa: BLE001 - a failed sample is evidence + row.update(outcome='error', code='harness', message=f'{type(error).__name__}: {error}'[:300]) + self.ended = time.monotonic() + self.keep(row) + + def stop(self): + self.stopped_at = time.monotonic() + self.halt.set() + self.join() + return self.rows + + +class ReleaseWatcher(threading.Thread): + """Times DevGuard's release of terminated targets: from the termination request until the + journal, read every 100 ms, shows the attempt holding nothing. One row per watched slot; an + attempt still charged after `limit` seconds is recorded unreleased.""" + + def __init__(self, bench, writer, path, limit=60.0): + super().__init__(daemon=True) + self.bench, self.writer, self.path, self.limit = bench, writer, path, limit + self.pending = {} + self.lock = threading.Lock() + self.halt = threading.Event() + self.rows = [] + self.read_errors = 0 + + def watch(self, slot, attempt, asked): + with self.lock: + self.pending[slot] = (attempt, asked) + + def poll(self, final=False): + with self.lock: + pending = dict(self.pending) + if not pending: + return + ids = [attempt for attempt, _ in pending.values() if attempt] + try: + found = host.journal_attempts(ids, self.bench.database, self.bench.authority['consumer']) if ids else {} + except Exception: # noqa: BLE001 - a busy journal is read again + self.read_errors += 1 + found = None + now = time.monotonic() + for slot, (attempt, asked) in sorted(pending.items()): + view = (found or {}).get(attempt) + if view is not None and not view['charged']: + released = round((now - asked) * 1000, 3) + elif final or not attempt or now - asked >= self.limit: + released = None + else: + continue + with self.lock: + self.pending.pop(slot, None) + row = {'slot': slot, 'attempt_id': attempt, 'released_ms': released, + 'phase': (view or {}).get('phase'), 'release_reason': (view or {}).get('release_reason')} + self.rows.append(row) + self.writer.write(self.path, row) + + def run(self): + while not self.halt.is_set(): + self.poll() + self.halt.wait(0.1) + + def finish(self, limit=60.0): + """Wait up to `limit` seconds for the pending releases; the rest are recorded unreleased.""" + deadline = time.monotonic() + limit + while self.pending and time.monotonic() < deadline and self.is_alive(): + time.sleep(0.2) + self.halt.set() + if self.is_alive(): + self.join() + self.poll(final=True) + return self.rows + + +def status_action(gateway, process_id): + def sample(slot): + payload, latency = gateway.status(process_id, timeout=10) + return {'outcome': 'answered', 'latency_ms': latency, 'state': payload.get('state')} + return sample + + +class Echo: + """A numbered line written with write_stdin to the running governed `cat`, timed until + read_process returns it.""" + + def __init__(self, gateway, process_id): + self.gateway, self.process_id, self.cursor = gateway, process_id, 0 + + def __call__(self, slot): + line = f'io-{slot}\n' + began = time.monotonic() + self.gateway.call('write_stdin', {'process_id': self.process_id, 'data': line}, timeout=10) + _, self.cursor, eof, found = self.gateway.read_until(self.process_id, line, limit=10, period=0.01, + cursor=self.cursor, timeout=10) + return {'outcome': 'echoed' if found else 'not_echoed', + 'latency_ms': round((time.monotonic() - began) * 1000, 3), 'eof': eof} + + +class Terminations: + """Each slot terminates one governed target and times it as DevGuard's protocol does: the + acknowledgement to terminate_process's answer; the target's end to its exited status and to + its scope's process group gone; DevGuard's release through the watcher. `target(slot)` gives + a started target, or the result of one that did not start: refused for capacity or pressure + (RESOURCE_UNAVAILABLE) it is not a sample, otherwise a missing one.""" + + def __init__(self, gateway, watcher, target): + self.gateway, self.watcher, self.target = gateway, watcher, target + + def __call__(self, slot): + started = self.target(slot) + if started.get('outcome') != 'started': + code = started.get('code') + return {'outcome': 'not_started' if code == 'RESOURCE_UNAVAILABLE' else 'error', 'code': code, + 'attempt_id': started.get('attempt_id'), 'latency_ms': started.get('latency_ms'), + 'tty': started.get('tty')} + pid = started['process_id'] + entry = self.gateway.execution(pid) or {} + root, attempt = entry.get('scope_root_pid'), entry.get('attempt_id') + asked = time.monotonic() + ack = self.gateway.terminate(pid, timeout=10) + self.watcher.watch(slot, attempt, asked) + status = self.gateway.wait_exited(pid, limit=30, period=0.02) + exited = round((time.monotonic() - asked) * 1000, 3) if status else None + scope = host.wait_group_gone(root, limit=30) if root else None + return {'outcome': 'acknowledged', 'tty': started.get('tty'), 'ack_ms': ack, + 'termination': (status or {}).get('termination'), 'exited_ms': exited, + 'scope_exit_ms': round((time.monotonic() - asked) * 1000, 3) if scope is not None else None, + 'attempt_id': attempt, 'scope_root_pid': root} + + +def fresh_target(gateway, workspace): + """A new governed `sleep` per slot, alternating pipe and PTY.""" + def target(slot): + tty = bool(slot % 2) + return dict(gateway.exec(workspace, ['/bin/sleep', '600'], tty=tty), tty=tty) + return target + + +def pooled_target(pool): + """Targets started before a fault, one per slot.""" + def target(slot): + if not pool: + return {'outcome': 'error', 'code': 'harness', 'message': 'no prepared target'} + return pool.pop(0) + return target + + +def admission_action(gateway, workspace): + """A new governed execution that the authority refuses or, where it admits, ends at once: + refused, admitted or a transport error, with the call's round trip.""" + def sample(slot): + result = gateway.exec(workspace, ['/usr/bin/true']) + row = {'latency_ms': result.get('latency_ms'), 'code': result.get('code'), 'attempt_id': result.get('attempt_id')} + if result['outcome'] == 'started': + row.update(outcome='admitted', attempt_id=attempt_of(gateway, result['process_id'])) + gateway.wait_exited(result['process_id'], limit=30) + else: + row['outcome'] = 'refused' if result['outcome'] == 'refused' else 'error' + return row + return sample + + +def end_target(gateway, process_id): + """Terminate a status target at the end of its interval: effective when it is reported + terminated and its scope's process group is gone.""" + root = (gateway.execution(process_id) or {}).get('scope_root_pid') + try: + ack = gateway.terminate(process_id, timeout=10) + status = gateway.wait_exited(process_id, limit=30) + except ToolError as error: + return {'process_id': process_id, 'error': error.code, 'effective': False} + scope = host.wait_group_gone(root, limit=30) if root else None + termination = (status or {}).get('termination') + return {'process_id': process_id, 'ack_ms': ack, 'termination': termination, 'scope_gone': scope is not None, + 'effective': termination == 'terminated' and scope is not None} + + +def read_rows(directory, name): + path = Path(directory) / f'{name}.jsonl' + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def raw_hashes(directory): + directory = Path(directory) + return {str(path.relative_to(directory)): host.sha256(path) for path in sorted(directory.rglob('*')) + if path.is_file() and path.suffix in ('.jsonl', '.json') and path.name not in ('interval.json', 'phase.json')} + + +# ---------------------------------------------------------------- authority (fixture authority faults) +# Control-path latency while the fixture authority is paused, stalled, restarted and stopped +# (plan ยง6, "Authority failure"): process control must not depend on the authority, and new +# governed executions must be refused promptly where it cannot answer. + +PHASE_S = 15.0 +PHASE_PERIODS = {'status': 1.0, 'termination': PHASE_S, 'io': 3.0} +PHASE_TERMINATIONS = (3.0, 7.0, 11.0) +PHASE_ADMISSIONS = (1.0, 5.0, 9.0, 13.0) +# Each phase's fault, what new governed executions must meet, and whether DevGuard can still +# release terminated targets (it cannot after a restart, which turns them Suspect, or once +# stopped). +AUTHORITY_PHASES = (('baseline', None, 'admitted', True), ('paused', 'pause', 'admitted', True), + ('stalled', 'hold', 'refused', True), ('restarted', 'restart', 'admitted', False), + ('stopped', 'kill', 'refused', False)) +REFUSED_UNREACHABLE = ('RESOURCE_AUTHORITY_UNAVAILABLE', 'ADMISSION_UNKNOWN') + + +def authority_workspaces(work): + governed = host.required(**FIXTURE_REQUEST) + return {name: (work / f'ws/{name}', governed) + for name in ('status-pipe', 'status-pty', 'admit', 'term-0', 'term-1', 'term-2')} + + +def phase_checks(expectation, admissions): + """Whether each new governed execution met the phase's expectation.""" + ran = [row for row in admissions if row.get('outcome') != 'missed_slot'] + if expectation == 'admitted': + passed = bool(ran) and all(row.get('outcome') == 'admitted' for row in ran) + else: + passed = bool(ran) and all(row.get('outcome') == 'refused' and row.get('code') in REFUSED_UNREACHABLE + for row in ran) + return [check(f'new governed executions are {expectation}', passed, + outcomes=[(row.get('outcome'), row.get('code'), row.get('latency_ms')) for row in ran])] + + +def phase_verdict(record, rows): + """A phase's verdict from its raw rows and recorded checks.""" + metrics = analysis.control_metrics(rows, record['seconds'], PHASE_PERIODS, record['release_expected'], + len(PHASE_TERMINATIONS)) + verdict, reasons = analysis.interval_verdict('fault', record['invalid'], metrics) + failed = [item['check'] for item in record['checks'] if item['passed'] is False] + if failed and verdict != 'inconclusive': + verdict, reasons = 'fail', reasons + failed + return verdict, reasons, metrics + + +def authority_phase(bench, gateway, evidence, mode, targets, echo, phase, fault, expectation, release_expected): + """One phase: its termination targets started before the fault, the fault applied, the + schedules run for PHASE_S seconds, then the fault lifted where it can be. The samples have + their own writer, closed before their files are hashed.""" + raw = evidence.file(f'{mode}/authority/{phase}/x').parent + samples = host.Writer() + schedules, watcher, invalid = {}, None, [] + try: + pool = [dict(gateway.exec(f'term-{n}', ['/bin/sleep', '600'], tty=bool(n % 2)), tty=bool(n % 2)) + for n in range(len(PHASE_TERMINATIONS))] + prepared = [entry['process_id'] for entry in pool if entry['outcome'] == 'started'] + if len(prepared) < len(pool): + invalid.append('a termination target could not be started before the fault') + watcher = ReleaseWatcher(bench, samples, raw / 'releases.jsonl', limit=60) + watcher.start() + applied = None + if fault == 'hold': + applied = bench.fixture.command(f'hold {int((PHASE_S + 1) * 1000)}') + elif fault == 'kill': + bench.fixture.kill() + applied = {'killed': True} + elif fault: + applied = bench.fixture.command(fault) + began = time.monotonic() + begin = began + 0.2 + started_unix = time.time() + schedules = { + 'status_pipe': Schedule(samples, raw / 'status_pipe.jsonl', status_action(gateway, targets['pipe']), begin, + period=PHASE_PERIODS['status']), + 'status_pty': Schedule(samples, raw / 'status_pty.jsonl', status_action(gateway, targets['pty']), begin, + period=PHASE_PERIODS['status']), + 'terminations': Schedule(samples, raw / 'terminations.jsonl', + Terminations(gateway, watcher, pooled_target(pool)), begin, + offsets=PHASE_TERMINATIONS), + 'refusals': Schedule(samples, raw / 'refusals.jsonl', admission_action(gateway, 'admit'), begin, + offsets=PHASE_ADMISSIONS), + 'io': Schedule(samples, raw / 'io.jsonl', echo, begin + 0.5, period=PHASE_PERIODS['io'])} + for schedule in schedules.values(): + schedule.start() + until(begin + PHASE_S) + rows = {name: schedule.stop() for name, schedule in schedules.items()} + seconds = round(time.monotonic() - begin, 3) + if fault == 'pause': + bench.fixture.command('resume') + elif fault == 'hold': + until(began + PHASE_S + 1.5) + rows['releases'] = watcher.finish(limit=30 if release_expected else 2) + except BaseException: + for thread in list(schedules.values()) + ([watcher] if watcher else []): + thread.halt.set() + samples.close(limit=10) + raise + for process_id in prepared: + try: + if gateway.status(process_id, timeout=10)[0].get('state') != 'exited': + end_target(gateway, process_id) + except ToolError: + pass + written = samples.close() + if written['error_count']: + invalid.append(f"raw evidence could not be written: {written['errors'][:3]}") + record = {'mode': mode, 'phase': phase, 'fault': fault, 'applied': applied, 'started_unix': started_unix, + 'seconds': seconds, 'release_expected': release_expected, 'expectation': expectation, + 'invalid': invalid, 'checks': phase_checks(expectation, rows['refusals']), + 'read_errors': watcher.read_errors, 'evidence_writer': written, 'raw': raw_hashes(raw)} + record['verdict'], record['reasons'], record['metrics'] = phase_verdict(record, rows) + evidence.write_json(f'{mode}/authority/{phase}/phase.json', record) + print(f"qualify: {mode} authority {phase}: {record['verdict']} {record['reasons']}", flush=True) + return record + + +SETUP_FAILED = ['the status targets could not be started'] + + +def authority_mode(bench, evidence, mode): + gateway = bench.gateway('control', timeout_s=3600) + pipe = gateway.exec('status-pipe', ['/bin/cat']) + pty = gateway.exec('status-pty', ['/bin/sleep', '86400'], tty=True) + if pipe['outcome'] != 'started' or pty['outcome'] != 'started': + record = {'mode': mode, 'phase': 'setup', 'verdict': 'inconclusive', 'reasons': SETUP_FAILED, + 'targets': [pipe, pty]} + evidence.write_json(f'{mode}/authority/setup/phase.json', record) + return [record] + targets = {'pipe': pipe['process_id'], 'pty': pty['process_id']} + echo = Echo(gateway, pipe['process_id']) + return [authority_phase(bench, gateway, evidence, mode, targets, echo, *phase) for phase in AUTHORITY_PHASES] + + +def authority(args, live, evidence): + if platform.system() != 'Darwin': + return {'verdict': 'incomplete', 'problems': ['the authority faults need native host evidence (macOS)'], + 'phases': []} + records = [] + for mode in args.modes: + writer = host.Writer() + bench = None + try: + bench = fixture_bench(args, live, evidence, writer, mode, 'authority', authority_workspaces) + records += authority_mode(bench, evidence, mode) + finally: + if bench is not None: + bench.teardown() + evidence.write_json(f'{mode}/writer.json', writer.close()) + verdict = analysis.suite_verdict([record['verdict'] for record in records]) + return {'verdict': verdict, 'phases': [{key: record[key] for key in ('mode', 'phase', 'verdict', 'reasons')} + for record in records]} + + +# ---------------------------------------------------------------- control (installed service, DG-1 load) +# Control-path latency of CodeSpace's gateway against the installed service while DevGuard's own +# development load runs (plan ยง6). The load is DG-1's six consumers from DevGuard's measure.py at +# the given checkout, run through the installed release's `devguard exec`, so pressure and queueing +# are the service's own; the targets fit with that load in half the work capacity, the Constrained +# capacity, as DG-1's own probe does. + +CONTROL_PROTOCOL = {'idle_s': 600, 'load_s': 1800, 'repetitions': 3, 'combinations': list(MODES)} +CONTROL_REHEARSAL = {'idle_s': 30, 'load_s': 60, 'repetitions': 1, 'combinations': list(MODES)} +PERIODS = {'status': 1.0, 'termination': 20.0, 'io': 5.0, 'refusal': 30.0} +HOST_PERIOD_S = 5.0 +SERVICE_PERIOD_S = 60.0 +FOOTPRINT_PERIOD_S = 5.0 +CONTROL_REQUEST = {'cpu_milli': 25, 'memory_bytes': 16 << 20, 'tasks': 2} +# Status targets run for a whole interval, the load's completion limit included. +CONTROL_TIMEOUT_S = 14400 + + +def load_measure(checkout): + """DevGuard's measure.py from a checkout, for DG-1's load consumers and their settlement.""" + scripts = Path(checkout).resolve() / 'scripts' + sys.path.insert(0, str(scripts)) + spec = importlib.util.spec_from_file_location('devguard_measure', scripts / 'measure.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def control_workspaces(work, capacity): + governed = host.required(**CONTROL_REQUEST) + hog = dict(CONTROL_REQUEST, cpu_milli=((capacity or {}).get('cpu_milli') or 1_000_000) + 1) + return {'status-pipe': (work / 'ws/status-pipe', governed), 'status-pty': (work / 'ws/status-pty', governed), + 'term': (work / 'ws/term', governed), 'hog': (work / 'ws/hog', host.required(**hog))} + + +def launched_count(database): + return sum(1 for view in host.journal_attempts(None, database).values() if view['launched']) + + +def journal_findings(database, gateway, rows, launched_before): + """The harness's attempts once settled: none still charged, none launched twice (the gateway's + log against the journal's count of launched attempts), and no uncertain attempt launched.""" + attempts = set(gateway.attempts()) + uncertain = set() + for name in ('refusals', 'terminations'): + for row in rows[name]: + if row.get('attempt_id'): + attempts.add(row['attempt_id']) + if row.get('code') == 'ADMISSION_UNKNOWN': + uncertain.add(row['attempt_id']) + try: + found, charged = host.settle_attempts(sorted(attempts), database, 60) + launched_after = launched_count(database) + except Exception as error: # noqa: BLE001 - an unreadable journal invalidates the interval + return {'error': f'{type(error).__name__}: {error}'} + launches = gateway.launches() + logged = sum(launches.values()) + in_journal = None if launched_before is None else launched_after - launched_before + unaccounted = max(0, in_journal - logged) if in_journal is not None else 0 + return {'harness_attempts': len(attempts), 'found': len(found), 'still_charged': charged, + 'launched_in_log': logged, 'launched_in_journal': in_journal, + 'duplicate_launches': sum(1 for count in launches.values() if count > 1) + unaccounted, + 'readmitted_uncertain': sum(1 for attempt in uncertain if (found.get(attempt) or {}).get('launched'))} + + +def footprint(owner, gateway, mode): + def sample(): + return {'owner': host.process_footprint(owner) if owner else None, + 'gateway': host.process_footprint(gateway.pid) if mode == 'uds' else None} + return sample + + +def interval_metrics(record, rows, sampled, runs, release_id, service_pid, reservation): + """A control interval's metrics from its raw rows, as the run computes them and `verify` + recomputes them.""" + metrics = analysis.control_metrics(rows, record['seconds'], PERIODS) + targets = record.get('status_targets') or [] + metrics['termination']['status_target_effective'] = (all(t.get('effective') for t in targets) + if targets else None) + _, failure = analysis.service_findings(sampled['service'], release_id, service_pid) + metrics['service'] = {'rows': len(sampled['service']), 'failure': failure} + metrics['host'] = analysis.host_peaks(sampled['host']) + metrics['footprint'] = { + 'owner': analysis.footprint_metrics([r['owner'] for r in sampled['footprint'] if r.get('owner')], reservation), + 'gateway': analysis.footprint_metrics([r['gateway'] for r in sampled['footprint'] if r.get('gateway')], None)} + metrics['journal'] = record.get('journal') or {} + if record['kind'] == 'load': + metrics['load'] = analysis.load_metrics(runs, record['started_unix'], record['ended_unix'], + record.get('consumers') or ()) + metrics['load']['issues'] += record.get('load_issues') or [] + metrics['load']['failures'] += record.get('load_failures') or [] + return metrics + + +def control_interval(context, mode, kind, seconds, key, service_pid): + args, live, evidence, measure, release, dg_release, work, reservation = ( + context[name] for name in ('args', 'live', 'evidence', 'measure', 'release', 'dg_release', 'work', 'reservation')) + prefix = f'{key}/{kind}' + raw = evidence.file(prefix + '/x').parent + record = {'kind': kind, 'combination': mode, 'invalid': []} + invalid = record['invalid'] + launched_before = None + try: + before = host.journal_charged(args.journal) + launched_before = launched_count(args.journal) + if sum(before['attempts'].values()) or before['leases']: + invalid.append("work other than the harness's was charged when the interval began") + except Exception as error: # noqa: BLE001 - an unreadable journal invalidates the interval + before = {'error': str(error)} + invalid.append(f'the journal could not be read: {error}') + record['journal_before'] = before + writer = host.Writer() + directory = scratch() + bench = Bench(args, live, evidence, writer, mode, directory, + control_workspaces(directory, host.work_capacity(args.service_log)), installed_authority(args), + prefix=prefix) + schedules, samplers, load, watcher, targets = {}, {}, [], None, {} + try: + gateway = bench.gateway('control', timeout_s=CONTROL_TIMEOUT_S) + record['resource_authority'], _ = bench.registration(gateway, 'status-pipe') + targets = {'pipe': gateway.exec('status-pipe', ['/bin/cat']), + 'pty': gateway.exec('status-pty', ['/bin/sleep', '86400'], tty=True)} + if any(target['outcome'] != 'started' for target in targets.values()): + invalid.append('the status targets could not be started: ' + + ', '.join(str(target.get('code')) for target in targets.values())) + watcher = ReleaseWatcher(bench, writer, raw / 'releases.jsonl') + watcher.start() + begin = time.monotonic() + 0.5 + for kind_, target in targets.items(): + if target['outcome'] == 'started': + schedules[f'status_{kind_}'] = Schedule(writer, raw / f'status_{kind_}.jsonl', + status_action(gateway, target['process_id']), begin, + period=PERIODS['status']) + if targets['pipe']['outcome'] == 'started': + schedules['io'] = Schedule(writer, raw / 'io.jsonl', Echo(gateway, targets['pipe']['process_id']), + begin + 0.25, period=PERIODS['io']) + schedules['terminations'] = Schedule(writer, raw / 'terminations.jsonl', + Terminations(gateway, watcher, fresh_target(gateway, 'term')), + begin + PERIODS['termination'] / 2, period=PERIODS['termination']) + schedules['refusals'] = Schedule(writer, raw / 'refusals.jsonl', admission_action(gateway, 'hog'), + begin + PERIODS['refusal'] / 3, period=PERIODS['refusal']) + samplers = {'host': host.Sampler(writer, raw / 'host.jsonl', HOST_PERIOD_S, host.host_sample), + 'service': host.Sampler(writer, raw / 'service.jsonl', SERVICE_PERIOD_S, release.state), + 'footprint': host.Sampler(writer, raw / 'footprint.jsonl', FOOTPRINT_PERIOD_S, + footprint(bench.owner_pid, gateway, mode))} + until(begin) + record['started_unix'] = time.time() + for thread in list(schedules.values()) + list(samplers.values()): + thread.start() + if kind == 'load': + (raw / 'runs').mkdir() + load = measure.consumers(SimpleNamespace(qualify_bin=Path(args.qualify_bin)), dg_release, args.load, work, + raw / 'runs', begin + seconds, key, args.rehearsal) + for consumer in load: + consumer.start() + limit = begin + seconds + measure.LOAD_COMPLETION_LIMIT_S + for consumer in load: + consumer.join(timeout=max(0.0, limit - time.monotonic())) + if any(consumer.is_alive() for consumer in load): + invalid.append('the load did not complete within the limit') + measure.settle_consumers(load) + else: + until(begin + seconds) + record['ended_unix'] = time.time() + record['seconds'] = round(time.monotonic() - begin, 3) + except BaseException: + for thread in list(schedules.values()) + list(samplers.values()) + ([watcher] if watcher else []): + thread.halt.set() + writer.close(limit=10) + bench.teardown() + raise + rows = {name: schedules[name].stop() if name in schedules else [] for name in analysis.SAMPLE_FILES + if name != 'releases'} + sampled = {name: sampler.stop() for name, sampler in samplers.items()} + record['status_targets'] = [end_target(gateway, target['process_id']) for target in targets.values() + if target['outcome'] == 'started'] + rows['releases'] = watcher.finish(limit=60) + bench.teardown() + record['journal'] = journal_findings(args.journal, gateway, rows, launched_before) + runs = [run for consumer in load for run in consumer.runs] + if kind == 'load': + (raw / 'runs.json').write_text(json.dumps(runs, indent=1) + '\n') + record['consumers'] = [consumer.name for consumer in load] + errors = {consumer.name: consumer.errors for consumer in load if consumer.errors} + record['load_issues'] = ['a consumer stopped with an error'] if errors else [] + record['consumer_errors'] = errors + ids = {attempt for run in runs for attempt in run.get('attempt_ids') or []} + record['load_journal'] = measure.settle_journal(ids, runs) + record['load_failures'] = (['load attempts still charged after the interval'] + if record['load_journal'].get('still_charged') else []) + written = writer.close() + record['evidence_writer'] = written + if written['error_count']: + invalid.append(f"raw evidence could not be written: {written['errors'][:3]}") + if 'error' in record['journal']: + invalid.append(f"the journal could not be read: {record['journal']['error']}") + service_invalid, _ = analysis.service_findings(sampled['service'], release.id, service_pid) + invalid += service_invalid + record['raw'] = raw_hashes(raw) + metrics = interval_metrics(record, rows, sampled, runs, release.id, service_pid, reservation) + record['verdict'], record['reasons'] = analysis.interval_verdict(kind, invalid, metrics) + record['metrics'] = metrics + evidence.write_json(prefix + '/interval.json', record) + print(f"qualify: {key} {kind}: {record['verdict']} {record['reasons']}", flush=True) + return record + + +def early_outcome(report): + """The verdict of a repetition that could not begin its intervals, or None.""" + if not report['service_before']['runs_release']: + return 'inconclusive', ['the service did not run the release'] + prewarm = report.get('prewarm') + if prewarm is not None and not any(run.get('result') == 'completed' and run.get('exit_code') == 0 + for run in prewarm): + return 'inconclusive', ['the warm target directory could not be built'] + return None + + +def repetition_outcome(report, idle, load): + """A repetition's verdict from its two intervals and the service before and after them.""" + verdict, reasons = analysis.repetition_verdict(idle, load) + before, after = report['service_before'], report['service_after'] + if verdict == 'pass' and not (after['runs_release'] and after['service_pid'] == before['service_pid']): + verdict, reasons = 'inconclusive', reasons + ['the service changed after the intervals'] + if report['rehearsal']: + verdict, reasons = 'inconclusive', reasons + ['rehearsal durations are below the protocol'] + return verdict, reasons + + +def control_repetition(context, plan, mode, repetition): + args, evidence, measure, release = (context[name] for name in ('args', 'evidence', 'measure', 'release')) + key = f'{mode}-{repetition}' + report = {'schema': 'codespace-control-repetition/v1', 'combination': mode, 'repetition': repetition, + 'rehearsal': args.rehearsal, 'load': args.load, 'service_before': release.state()} + service_pid = report['service_before']['service_pid'] + + def conclude(outcome): + report['verdict'], report['reasons'] = outcome + evidence.write_json(f'{key}/report.json', report) + print(f"qualify: {key} {report['verdict']} {report['reasons']}", flush=True) + return report + + if report['service_before']['runs_release'] and args.load == 'warm': + report['prewarm'] = measure.prewarm(SimpleNamespace(qualify_bin=Path(args.qualify_bin)), context['dg_release'], + context['work'], evidence.file(f'{key}/x').parent) + early = early_outcome(report) + if early: + report['service_after'] = release.state() + return conclude(early) + idle = control_interval(context, mode, 'idle', plan['idle_s'], key, service_pid) + load = control_interval(context, mode, 'load', plan['load_s'], key, service_pid) + report['intervals'] = {'idle': {k: idle[k] for k in ('verdict', 'reasons')}, + 'load': {k: load[k] for k in ('verdict', 'reasons')}} + for target in sorted((context['work'] / 'targets.noindex').glob(f'cold-{key}-*')): + shutil.rmtree(target, ignore_errors=True) + report['service_after'] = release.state() + return conclude(repetition_outcome(report, idle, load)) + + +def control_preconditions(args, release, measure): + problems = [] + state = release.state() + if not state['runs_release']: + problems.append(f'the service does not run {release.id}') + charged = host.journal_charged(args.journal) + if sum(charged['attempts'].values()) or charged['leases']: + problems.append(f'the service has charged work: {charged}') + doctor = release.doctor() + if not doctor.get('satisfied'): + problems.append('devguard doctor is not satisfied') + config = Path(args.host_config) + if (consumer_settings(config.read_text()) if config.exists() else {}).get('role') != 'control_service': + problems.append('host.toml has no codespace control-service consumer') + if not (host.run_text('rustc', '--version') or '').startswith(f'rustc {PINNED_RUST} '): + problems.append(f'the Cargo load needs Rust {PINNED_RUST} first in PATH') + if not Path(args.qualify_bin).is_file(): + problems.append('--qualify-bin is not a file') + commit = json.loads(release.manifest.read_text())['source']['commit'] + if host.run_text('git', '-C', measure.ROOT, 'cat-file', '-e', f'{commit}^{{commit}}') is None: + problems.append(f"the DevGuard checkout does not have the release's source commit {commit}") + return problems + + +def control(args, live, evidence): + if platform.system() != 'Darwin': + return {'verdict': 'inconclusive', 'problems': ['the control measurement runs only on macOS']} + measure = load_measure(args.devguard_checkout) + live.measure = measure + release = host.Release(args.release) + plan = dict(CONTROL_REHEARSAL if args.rehearsal else CONTROL_PROTOCOL, combinations=list(args.modes), + load=args.load, rehearsal=args.rehearsal) + if args.repetitions is not None: + plan['repetitions'] = args.repetitions + problems = control_preconditions(args, release, measure) + work = Path(args.work).resolve() + if work.exists(): + problems.append(f'{work} already exists') + if problems: + return {'verdict': 'inconclusive', 'problems': problems} + work.mkdir(parents=True) + manifest = json.loads(release.manifest.read_text()) + measure.export_source(manifest['source']['commit'], work) + config = Path(args.host_config) + reservation = consumer_settings(config.read_text()).get('control_reservation') + evidence.write_json('run.json', { + 'schema': 'codespace-control-run/v1', 'argv': sys.argv, 'plan': plan, 'protocol': CONTROL_PROTOCOL, + 'periods': PERIODS, 'request': CONTROL_REQUEST, 'reservation': reservation, + 'targets': {'status_p99_ms': analysis.STATUS_P99_MS, 'termination_ack_p99_ms': analysis.TERMINATION_P99_MS}, + 'release': {'id': release.id, **release.artifacts(), 'state': release.state()}, 'policy': policy(release, args), + 'work_capacity': host.work_capacity(args.service_log), 'host': host.host_facts(), + 'harness': {'commit': git('rev-parse', 'HEAD'), 'dirty': dirty(), + 'measure_sha256': host.sha256(Path(measure.__file__)), + 'devguard_checkout': git('rev-parse', 'HEAD', root=measure.ROOT), + 'qualify_sha256': host.sha256(args.qualify_bin), 'rustc': host.run_text('rustc', '--version')}, + 'started_unix': time.time()}) + context = {'args': args, 'live': live, 'evidence': evidence, 'measure': measure, 'release': release, + 'dg_release': measure.Release(args.release), 'work': work, 'reservation': reservation} + # The display and the system stay awake for the run; caffeinate ends with this process. + awake = subprocess.Popen(['/usr/bin/caffeinate', '-d', '-i', '-w', str(os.getpid())], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + live.workers.append(awake) + results = {} + try: + for mode in plan['combinations']: + results[mode] = [control_repetition(context, plan, mode, repetition) + for repetition in range(1, plan['repetitions'] + 1)] + finally: + awake.terminate() + combinations = {} + for mode, reports in results.items(): + combinations[mode] = { + 'verdict': analysis.combination_verdict([report['verdict'] for report in reports], plan['repetitions']), + 'repetitions': [{'repetition': report['repetition'], 'verdict': report['verdict'], + 'reasons': report['reasons'], + 'report_sha256': host.sha256(evidence.path / f"{mode}-{report['repetition']}/report.json")} + for report in reports]} + return {'verdict': analysis.overall_verdict(combinations, plan, CONTROL_PROTOCOL, args.rehearsal), + 'plan': plan, 'combinations': combinations} + + +# ---------------------------------------------------------------- verify + +def normalized(value): + """A value as its JSON evidence holds it.""" + return json.loads(json.dumps(value)) + + +def read_json(path): + return json.loads(Path(path).read_text()) + + +def compare(problems, label, recorded, recomputed): + """Record a problem when a recorded value differs from its recomputation.""" + recorded, recomputed = normalized(recorded), normalized(recomputed) + if recorded != recomputed: + problems.append(f'{label}: recorded {json.dumps(recorded)[:400]}, recomputed {json.dumps(recomputed)[:400]}') + + +def verify_candidate(directory, problems): + record = read_json(directory / 'candidate.json') + found = candidate_problems(record) + report = read_json(directory / 'report.json') + if 'result' in report: + compare(problems, 'candidate problems', report['result'].get('problems'), found) + return 'failed' if found else 'passed' + + +def verify_scenarios(directory, problems): + """matrix and faults: each scenario's verdict from its recorded checks.""" + verdicts = [] + for entry in read_rows(directory, 'scenarios'): + verdict = analysis.scenario_verdict(entry['checks']) + compare(problems, f"{entry['mode']} {entry['io'] or '-'} {entry['scenario']} verdict", entry['verdict'], verdict) + verdicts.append(verdict) + return analysis.suite_verdict(verdicts) + + +def verify_authority(directory, problems): + """Each authority phase's checks, metrics and verdict from its raw samples.""" + verdicts = [] + for path in sorted(directory.glob('*/authority/*/phase.json')): + record = read_json(path) + label = f"{record['mode']} authority {record['phase']}" + if record['phase'] == 'setup': + compare(problems, f'{label} verdict', [record['verdict'], record['reasons']], ['inconclusive', SETUP_FAILED]) + verdicts.append('inconclusive') + continue + rows = {name: read_rows(path.parent, name) for name in analysis.SAMPLE_FILES} + compare(problems, f'{label} raw files', record['raw'], raw_hashes(path.parent)) + compare(problems, f'{label} checks', record['checks'], phase_checks(record['expectation'], rows['refusals'])) + verdict, reasons, metrics = phase_verdict(record, rows) + compare(problems, f'{label} metrics', record['metrics'], metrics) + compare(problems, f'{label} verdict', [record['verdict'], record['reasons']], [verdict, reasons]) + verdicts.append(verdict) + return analysis.suite_verdict(verdicts) + + +def verify_interval(directory, key, kind, release_id, service_pid, reservation, problems): + """A control interval's metrics and verdict from its raw samples.""" + raw = directory / key / kind + record = read_json(raw / 'interval.json') + label = f'{key} {kind}' + rows = {name: read_rows(raw, name) for name in analysis.SAMPLE_FILES} + sampled = {name: read_rows(raw, name) for name in ('host', 'service', 'footprint')} + runs = read_json(raw / 'runs.json') if (raw / 'runs.json').exists() else [] + compare(problems, f'{label} raw files', record['raw'], raw_hashes(raw)) + service_invalid, _ = analysis.service_findings(sampled['service'], release_id, service_pid) + for reason in service_invalid: + if reason not in record['invalid']: + problems.append(f'{label}: the service samples make it invalid ({reason}), the record does not say so') + metrics = interval_metrics(record, rows, sampled, runs, release_id, service_pid, reservation) + verdict, reasons = analysis.interval_verdict(kind, record['invalid'], metrics) + compare(problems, f'{label} metrics', record['metrics'], metrics) + compare(problems, f'{label} verdict', [record['verdict'], record['reasons']], [verdict, reasons]) + return {'verdict': verdict, 'reasons': reasons} + + +def verify_control(directory, problems): + """Every interval from its raw samples, then each repetition, combination and the run.""" + if not (directory / 'run.json').exists(): + # The run stopped at its preconditions, before any interval. + return 'inconclusive' + run = read_json(directory / 'run.json') + plan, release_id, reservation = run['plan'], run['release']['id'], run['reservation'] + compare(problems, 'the run protocol', run['protocol'], CONTROL_PROTOCOL) + compare(problems, 'the targets', run['targets'], {'status_p99_ms': analysis.STATUS_P99_MS, + 'termination_ack_p99_ms': analysis.TERMINATION_P99_MS}) + combinations = {} + for mode in plan['combinations']: + verdicts = [] + for repetition in range(1, plan['repetitions'] + 1): + key = f'{mode}-{repetition}' + if not (directory / key / 'report.json').exists(): + continue + report = read_json(directory / key / 'report.json') + if 'intervals' in report: + service_pid = report['service_before']['service_pid'] + idle = verify_interval(directory, key, 'idle', release_id, service_pid, reservation, problems) + load = verify_interval(directory, key, 'load', release_id, service_pid, reservation, problems) + outcome = repetition_outcome(report, idle, load) + else: + outcome = early_outcome(report) or ('inconclusive', ['the repetition has no intervals']) + compare(problems, f'{key} verdict', [report['verdict'], report['reasons']], list(outcome)) + verdicts.append(outcome[0]) + combinations[mode] = {'verdict': analysis.combination_verdict(verdicts, plan['repetitions'])} + report = read_json(directory / 'report.json') + if 'result' in report: + compare(problems, 'combination verdicts', + {mode: entry['verdict'] for mode, entry in report['result']['combinations'].items()}, + {mode: entry['verdict'] for mode, entry in combinations.items()}) + for mode, entry in report['result']['combinations'].items(): + for repetition in entry['repetitions']: + path = directory / f"{mode}-{repetition['repetition']}" / 'report.json' + if not path.exists() or host.sha256(path) != repetition['report_sha256']: + problems.append(f"{mode}-{repetition['repetition']}: the report differs from the one the run recorded") + return analysis.overall_verdict(combinations, plan, CONTROL_PROTOCOL, plan['rehearsal']) + + +VERIFIERS = {'candidate': verify_candidate, 'matrix': verify_scenarios, 'faults': verify_scenarios, + 'authority': verify_authority, 'control': verify_control} +HARNESS = ('qualify-devguard.py', 'qualify_analysis.py', 'qualify_host.py') + + +def harness(): + """The harness that ran: its files' SHA-256 and the CodeSpace commit.""" + found = {'files': {name: host.sha256(ROOT / 'scripts' / name) for name in HARNESS}} + try: + found.update(commit=git('rev-parse', 'HEAD'), dirty=dirty()) + except (OSError, subprocess.CalledProcessError): + found.update(commit=None, dirty=None) + return found + + +def verify(directory): + """Check a sealed run's files against its MANIFEST.json and recompute its verdicts from the + raw evidence with this harness's analysis. Writes nothing.""" + directory = Path(directory).resolve() + if not (directory / 'MANIFEST.json').is_file() or not (directory / 'report.json').is_file(): + return {'directory': str(directory), 'verdict': 'failed', + 'problems': ['not a sealed run: MANIFEST.json or report.json is missing']} + problems = host.verify_manifest(directory) + report = read_json(directory / 'report.json') + suite = report.get('suite') + if suite not in VERIFIERS: + return {'directory': str(directory), 'verdict': 'failed', 'problems': problems + [f'unknown suite {suite!r}']} + try: + recomputed = VERIFIERS[suite](directory, problems) + except (OSError, ValueError, KeyError, TypeError) as error: + problems.append(f'the evidence cannot be recomputed: {type(error).__name__}: {error}') + recomputed = None + recorded = report.get('verdict') + if 'result' in report and recomputed is not None and recomputed != recorded: + problems.append(f'verdict: recorded {recorded}, recomputed {recomputed}') + ran = (report.get('harness') or {}).get('files') + return {'directory': str(directory), 'suite': suite, 'recorded_verdict': recorded, + 'recomputed_verdict': recomputed, 'same_harness': ran == harness()['files'], + 'verdict': 'failed' if problems else 'verified', 'problems': problems} + + +# ---------------------------------------------------------------- command line + +SUITES = {'candidate': candidate, 'matrix': matrix, 'faults': faults, 'authority': authority, 'control': control} + + +def modes(text): + chosen = [part.strip() for part in text.split(',') if part.strip()] + unknown = [mode for mode in chosen if mode not in MODES] + if unknown or not chosen: + raise argparse.ArgumentTypeError(f"modes are a comma-separated subset of {', '.join(MODES)}") + return chosen + + +def arguments(argv=None): + parser = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0], + epilog='docs/cs-rg-qualification.md describes each suite and its evidence.') + suites = parser.add_subparsers(dest='suite', metavar='suite') + suites.required = True + + common = argparse.ArgumentParser(add_help=False) + common.add_argument('--out', type=Path, required=True, help='a new evidence directory') + common.add_argument('--mcp-bin', type=Path, required=True, help='codespace-mcp built with --features devguard') + common.add_argument('--runtime-bin', type=Path, required=True, + help='codespace-codex-runtime built with --features devguard (the uds worker)') + common.add_argument('--modes', type=modes, default=list(MODES), help='in-process, uds or both (default both)') + installed = argparse.ArgumentParser(add_help=False) + installed.add_argument('--release', required=True, help='the installed DevGuard release ID') + installed.add_argument('--socket', type=Path, default=host.default_socket()) + installed.add_argument('--credential-file', type=Path, default=host.CREDENTIAL_FILE, + help="the codespace consumer's credential file; its path is passed, never read here") + installed.add_argument('--journal', type=Path, default=host.JOURNAL) + installed.add_argument('--service-log', type=Path, default=host.SERVICE_LOG) + installed.add_argument('--host-config', type=Path, default=host.HOST_CONFIG) + fixture = argparse.ArgumentParser(add_help=False) + fixture.add_argument('--fixture-bin', type=Path, required=True, + help="DevGuard's fixture authority at CodeSpace's pin (examples/fixture_authority)") + fixture.add_argument('--fixture-helper', type=Path, required=True, + help="devguard-launch built from CodeSpace's DevGuard pin") + + one = suites.add_parser('candidate', parents=[common, installed], help='record the combination under test') + for name in ('--fixture-bin', '--fixture-helper', '--qualify-bin'): + one.add_argument(name, type=Path, help='record this binary too') + suites.add_parser('matrix', parents=[common, installed], help='the claimed modes against the installed service') + suites.add_parser('faults', parents=[common, fixture], help="fault cases against DevGuard's fixture authority") + suites.add_parser('authority', parents=[common, fixture], + help='control-path latency while the fixture authority stalls, pauses, restarts and stops') + control_parser = suites.add_parser('control', parents=[common, installed], + help="control-path latency under DG-1's load against the installed service") + control_parser.add_argument('--work', type=Path, required=True, + help="a new directory for the load's DevGuard source and target directories") + control_parser.add_argument('--qualify-bin', type=Path, required=True, + help="DevGuard's devguard-qualify, as DG-1's measurement uses it") + control_parser.add_argument('--devguard-checkout', type=Path, required=True, + help="a DevGuard checkout holding the release's source commit, for scripts/measure.py") + control_parser.add_argument('--load', choices=('cold', 'warm'), default='cold') + control_parser.add_argument('--rehearsal', action='store_true', + help='short intervals to try the run; never qualifies') + control_parser.add_argument('--repetitions', type=int, help='repetitions per combination (protocol: 3)') + check_parser = suites.add_parser('verify', help='check a sealed run and recompute its verdicts') + check_parser.add_argument('directory', type=Path) + return parser.parse_args(argv) + + +def interrupted(number, frame): + raise KeyboardInterrupt(f'signal {number}') + + +def main(argv=None): + args = arguments(argv) + if args.suite == 'verify': + result = verify(args.directory) + print(json.dumps(result, indent=2)) + return 0 if result['verdict'] == 'verified' else 1 + evidence = host.Evidence(args.out) + live = Live() + for number in (signal.SIGTERM, signal.SIGHUP): + signal.signal(number, interrupted) + report = {'schema': 'codespace-qualification-report/v1', 'suite': args.suite, + 'argv': sys.argv if argv is None else [sys.argv[0], *argv], 'started_unix': time.time(), + 'harness': harness()} + status = EXIT_CRASH + try: + result = SUITES[args.suite](args, live, evidence) + report.update(result=result, verdict=result['verdict']) + status = EXIT[result['verdict']] + except KeyboardInterrupt: + report['verdict'] = 'interrupted' + status = 130 + except Exception as error: # noqa: BLE001 - a harness failure is recorded and sealed too + report.update(verdict='harness_failed', error=f'{type(error).__name__}: {error}', + trace=traceback.format_exc()[-5000:]) + finally: + report['settlement'] = live.settle() + report['ended_unix'] = time.time() + evidence.write_json('report.json', report) + manifest = evidence.seal() + print(f"qualify: {args.suite}: {report['verdict']}; evidence {evidence.path}, MANIFEST.json SHA-256 {manifest}", + flush=True) + return status + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/qualify_analysis.py b/scripts/qualify_analysis.py new file mode 100644 index 0000000..c4af30f --- /dev/null +++ b/scripts/qualify_analysis.py @@ -0,0 +1,358 @@ +"""CSRG-U6 qualification analysis: pure functions over recorded samples, so that every verdict +can be recomputed from the raw evidence alone. Targets and rules follow DevGuard's SLO protocol +(DevGuard docs/planning/verification.md and scripts/measure.py) wherever they apply; nothing here +redefines them.""" +import math + +# The established targets (DevGuard docs/planning/verification.md, "SLO protocol"). +STATUS_P99_MS = 500 +TERMINATION_P99_MS = 1000 +# Fewer samples than this share of an interval's schedule is not a complete observation. +MIN_SAMPLES = 0.9 +# Terminations whose targets were refused admission for capacity or pressure may lower this share. +MIN_TERMINATIONS = 0.8 +# Admitted load must run for at least this share of the load interval, and each consumer for at +# least its own share, with at least one successful run. +MIN_LOAD = 0.5 +MIN_CONSUMER = 0.25 +# The outcome code of a sample whose MCP connection was lost. +TRANSPORT = 'connection_lost' + + +def nearest_rank(values, fraction, missing=0): + """Nearest-rank percentile of raw values. A missing sample ranks above every value.""" + count = len(values) + missing + if count == 0: + return None + ordered = sorted(values) + [math.inf] * missing + return ordered[max(1, math.ceil(fraction * count)) - 1] + + +def number(value): + """A JSON-safe metric: a missing sample (infinite) is reported as the string "missing".""" + if value is None: + return None + return 'missing' if math.isinf(value) else round(value, 3) + + +def worst(values, missing): + return number(math.inf if missing else max(values, default=None)) + + +def status_metrics(samples, expected): + """`process_status` of a running governed target. An answer about a target that no longer + runs, an error and a missed slot are missing samples; a lost connection fails.""" + answered = [s['latency_ms'] for s in samples if s['outcome'] == 'answered' and s.get('state') == 'running'] + not_running = sum(1 for s in samples if s['outcome'] == 'answered' and s.get('state') != 'running') + errors = [s for s in samples if s['outcome'] == 'error'] + missed = sum(1 for s in samples if s['outcome'] == 'missed_slot') + losses = sum(1 for s in errors if s.get('code') == TRANSPORT) + missing = not_running + len(errors) + missed + p99 = nearest_rank(answered, 0.99, missing) + return {'samples': len(answered) + missing, 'expected': expected, 'answered': len(answered), + 'not_running': not_running, 'errors': len(errors), 'missed_slots': missed, + 'connection_losses': losses, 'p99_ms': number(p99), 'max_ms': worst(answered, missing), + 'passed': None if p99 is None else (p99 <= STATUS_P99_MS and losses == 0)} + + +def effective(sample): + """An acknowledged termination after which CodeSpace reported the target terminated and its + process group was observed gone.""" + return (sample['outcome'] == 'acknowledged' and sample.get('termination') == 'terminated' + and sample.get('scope_exit_ms') is not None) + + +def termination_metrics(samples, expected, release_expected=True): + """`terminate_process` of fresh governed targets. The acknowledgement is timed to the tool's + answer; the scope's actual end (its process group gone) and DevGuard's release are reported + separately and never combined with it. An acknowledgement that did not end its target fails + outright; an error or a missed slot is a missing sample, a lost connection fails. A target + refused admission for capacity or pressure is not a sample. Where DevGuard cannot release + (its authority restarted or stopped), `release_expected` is false and an unreleased target + is reported, not failed.""" + good = [s for s in samples if effective(s)] + ineffective = sum(1 for s in samples if s['outcome'] == 'acknowledged' and not effective(s)) + errors = [s for s in samples if s['outcome'] == 'error'] + missed = sum(1 for s in samples if s['outcome'] == 'missed_slot') + unstarted = sum(1 for s in samples if s['outcome'] == 'not_started') + losses = sum(1 for s in errors if s.get('code') == TRANSPORT) + missing = ineffective + len(errors) + missed + acks = [s['ack_ms'] for s in good] + exits = [s['scope_exit_ms'] for s in good] + released = [s['released_ms'] for s in good if s.get('released_ms') is not None] + unreleased = len(good) - len(released) + p99 = nearest_rank(acks, 0.99, missing) + return {'samples': len(good) + missing, 'expected': expected, 'effective': len(good), + 'ineffective': ineffective, 'errors': len(errors), 'missed_slots': missed, 'not_started': unstarted, + 'connection_losses': losses, 'ack_p99_ms': number(p99), 'ack_max_ms': worst(acks, missing), + 'scope_exit_p99_ms': number(nearest_rank(exits, 0.99)), 'scope_exit_max_ms': number(max(exits, default=None)), + 'release_p99_ms': number(nearest_rank(released, 0.99, unreleased)), + 'release_max_ms': worst(released, unreleased), 'unreleased': unreleased, + 'passed': (False if losses or ineffective or (unreleased and release_expected) + else None if p99 is None else p99 <= TERMINATION_P99_MS)} + + +def io_metrics(samples, expected): + """Output and control responsiveness: a line written with `write_stdin` to a running governed + `cat` until `read_process` returns it. There is no established numeric target, so the values + are reported; a lost connection or a line that never came back fails.""" + echoed = [s['latency_ms'] for s in samples if s['outcome'] == 'echoed'] + lost_lines = sum(1 for s in samples if s['outcome'] == 'not_echoed') + errors = [s for s in samples if s['outcome'] == 'error'] + missed = sum(1 for s in samples if s['outcome'] == 'missed_slot') + losses = sum(1 for s in errors if s.get('code') == TRANSPORT) + missing = lost_lines + len(errors) + missed + return {'samples': len(echoed) + missing, 'expected': expected, 'echoed': len(echoed), + 'not_echoed': lost_lines, 'errors': len(errors), 'missed_slots': missed, 'connection_losses': losses, + 'p50_ms': number(nearest_rank(echoed, 0.5, missing)), 'p99_ms': number(nearest_rank(echoed, 0.99, missing)), + 'max_ms': worst(echoed, missing), + 'passed': False if losses or lost_lines else (None if not samples else True)} + + +def refusal_metrics(samples): + """New-admission rejection latency: governed `exec_command` calls the authority must refuse + (a shortage, Critical pressure or an unavailable authority). Reported against the design's + 250 ms preparation budget, which is a reference and not an established target. A call that + ran instead of being refused is counted, and a lost connection fails.""" + refused = [s['latency_ms'] for s in samples if s['outcome'] == 'refused'] + codes = {} + for sample in samples: + if sample['outcome'] == 'refused': + codes[sample.get('code')] = codes.get(sample.get('code'), 0) + 1 + errors = [s for s in samples if s['outcome'] == 'error'] + losses = sum(1 for s in errors if s.get('code') == TRANSPORT) + admitted = sum(1 for s in samples if s['outcome'] == 'admitted') + return {'samples': len(samples), 'refused': len(refused), 'codes': codes, 'admitted': admitted, + 'errors': len(errors), 'connection_losses': losses, + 'p50_ms': number(nearest_rank(refused, 0.5)), 'p99_ms': number(nearest_rank(refused, 0.99)), + 'max_ms': number(max(refused, default=None)), + 'passed': False if losses else (None if not refused else True)} + + +def with_releases(terminations, releases): + """Each termination sample with the release time the journal watcher recorded for its slot.""" + found = {row.get('slot'): row.get('released_ms') for row in releases} + return [dict(row, released_ms=found.get(row.get('slot'))) for row in terminations] + + +def connection_lost(*row_lists): + """Whether any sample lost the MCP connection.""" + return any(row.get('code') == TRANSPORT for rows in row_lists for row in rows) + + +# The raw sample files of a control interval, one JSON line per sample. +SAMPLE_FILES = ('status_pipe', 'status_pty', 'terminations', 'releases', 'io', 'refusals') + + +def control_metrics(rows, elapsed, periods, release_expected=True, terminations_expected=None): + """The control path's metrics from an interval's raw samples: `process_status` of a running + governed pipe target and PTY target, terminations with their releases, I/O through the pipe + target and governed requests the authority must refuse. `periods` gives each schedule's + period in seconds; slots that never ran count against it.""" + status_expected = int(elapsed / periods['status']) + if terminations_expected is None: + terminations_expected = max(1, int(elapsed / periods['termination'])) + terminations = with_releases(rows['terminations'], rows['releases']) + return { + 'status': status_metrics(rows['status_pipe'] + rows['status_pty'], 2 * status_expected), + 'status_pipe': status_metrics(rows['status_pipe'], status_expected), + 'status_pty': status_metrics(rows['status_pty'], status_expected), + 'termination': termination_metrics(terminations, terminations_expected, release_expected), + 'io': io_metrics(rows['io'], int(elapsed / periods['io'])), + 'refusal': refusal_metrics(rows['refusals']), + 'connection_lost': connection_lost(*(rows[name] for name in SAMPLE_FILES)), + } + + +def covered(spans, started, ended): + """The share of [started, ended] covered by the union of spans.""" + total, cursor = 0.0, started + for begin, end in sorted((max(b, started), min(e, ended)) for b, e in spans): + begin = max(begin, cursor) + if end > begin: + total += end - begin + cursor = end + return round(total / max(ended - started, 1e-9), 4) + + +def load_metrics(runs, started, ended, consumers=()): + """What each load consumer did, the share of the interval in which its admitted work ran, and + what makes the load not the declared one (issues) or wrong (failures), as DevGuard's protocol + judges its own load.""" + entries = {name: [] for name in consumers} + for run in runs: + entries.setdefault(run['consumer'], []).append(run) + report, issues = {}, [] + for name, own in sorted(entries.items()): + ok = [r for r in own if r.get('result') == 'completed' and r.get('exit_code') == 0] + spans = [(r['running_from'], r['ended']) for r in own if r.get('running_from') is not None] + entry = {'runs': len(own), 'succeeded': len(ok), + 'nonzero_exit': sum(1 for r in own if r.get('result') == 'completed' and r.get('exit_code') != 0), + 'exec_failed': sum(1 for r in own if r.get('result') == 'exec_failed'), + 'not_started': sum(1 for r in own if r.get('result') == 'not_started'), + 'uncertain': sum(1 for r in own if r.get('result') == 'uncertain'), + 'unknown': sum(1 for r in own if r.get('result') not in + ('completed', 'exec_failed', 'not_started', 'uncertain')), + 'share': covered(spans, started, ended)} + report[name] = entry + if not ok: + issues.append(f'{name}: no successful run') + if entry['share'] < MIN_CONSUMER: + issues.append(f"{name}: ran for {entry['share']:.0%} of the interval") + for field in ('nonzero_exit', 'exec_failed', 'unknown'): + if entry[field]: + issues.append(f"{name}: {entry[field]} {field.replace('_', ' ')}") + applied = covered([(r['running_from'], r['ended']) for r in runs if r.get('running_from') is not None], + started, ended) + if applied < MIN_LOAD: + issues.append(f'admitted load ran for {applied:.0%} of the interval') + uncertain = sum(entry['uncertain'] for entry in report.values()) + failures = [f'{uncertain} load runs are uncertain'] if uncertain else [] + return {'consumers': report, 'applied_fraction': applied, 'uncertain': uncertain, + 'issues': issues, 'failures': failures} + + +def footprint_metrics(rows, reservation): + """The execution owner's measured footprint (the gateway, and in `uds` mode its worker) against + the static control reservation DevGuard holds for it. Reported, never a verdict: the + reservation is an accounting estimate.""" + def peak(field): + values = [row[field] for row in rows if row.get(field) is not None] + return max(values, default=None) + cpu = [row['cpu_percent'] for row in rows if row.get('cpu_percent') is not None] + result = {'samples': len(rows), 'rss_bytes_max': peak('rss_bytes'), 'threads_max': peak('threads'), + 'processes_max': peak('processes'), + 'cpu_milli_mean': round(10 * sum(cpu) / len(cpu), 1) if cpu else None, + 'cpu_milli_max': round(10 * max(cpu), 1) if cpu else None, 'reservation': reservation} + if reservation: + result['within_reservation'] = { + 'memory': None if result['rss_bytes_max'] is None else result['rss_bytes_max'] <= reservation['memory_bytes'], + 'tasks': None if result['threads_max'] is None else result['threads_max'] <= reservation['tasks'], + 'cpu_mean': None if result['cpu_milli_mean'] is None else result['cpu_milli_mean'] <= reservation['cpu_milli']} + return result + + +def host_peaks(rows): + """Memory pressure levels, swap and load over an interval's host samples.""" + levels = {} + for row in rows: + levels[str(row.get('pressure_level'))] = levels.get(str(row.get('pressure_level')), 0) + 1 + swap = [row['swap_used_mib'] for row in rows if row.get('swap_used_mib') is not None] + load = [row['load_average'][0] for row in rows if row.get('load_average')] + return {'samples': len(rows), 'pressure_levels': levels, 'swap_used_mib_max': max(swap, default=None), + 'load_average_max': max(load, default=None)} + + +def service_findings(rows, release_id, service_pid): + """(invalid, failure) from the service checks, as DevGuard's protocol judges them: another + release selected or a check that could not be made is invalid observation; the measured + release restarting or unhealthy is a failure.""" + invalid = [] + if not rows: + invalid.append('the service was not observed') + if any(row.get('error') for row in rows): + invalid.append('a service check could not be made') + if any(row.get('current') not in (None, release_id) for row in rows): + invalid.append('another release was selected during the interval') + ours = [row for row in rows if not row.get('error') and row.get('current') in (None, release_id)] + failure = None + if any(not row.get('runs_release') or row.get('service_pid') != service_pid for row in ours): + failure = 'the release restarted or was unhealthy during the interval' + return invalid, failure + + +def interval_verdict(kind, invalid, metrics): + """inconclusive when the observation is invalid; otherwise fail when a target was missed or + the accounting is wrong, and inconclusive when too little was observed or the load was not + the declared one.""" + if invalid: + return 'inconclusive', list(invalid) + names = ('status', 'termination', 'io', 'refusal') + failures = [name for name in names if metrics[name]['passed'] is False] + if metrics.get('connection_lost'): + failures.append('the MCP connection was lost') + if (metrics.get('service') or {}).get('failure'): + failures.append(metrics['service']['failure']) + if metrics['termination'].get('status_target_effective') is False: + failures.append("a status target's termination was not effective") + if kind == 'load': + failures += metrics['load']['failures'] + journal = metrics.get('journal') or {} + if journal.get('still_charged'): + failures.append('harness attempts still charged after the interval') + if journal.get('duplicate_launches'): + failures.append('an execution was launched more than once') + if journal.get('readmitted_uncertain'): + failures.append('an uncertain execution was admitted again') + if failures: + return 'fail', failures + shortfalls = [f'{name}: nothing was measured' for name in ('status', 'termination') if metrics[name]['passed'] is None] + status, termination = metrics['status'], metrics['termination'] + if status['samples'] < MIN_SAMPLES * status['expected']: + shortfalls.append(f"status samples {status['samples']} of {status['expected']}") + if termination['samples'] < MIN_TERMINATIONS * termination['expected']: + shortfalls.append(f"termination samples {termination['samples']} of {termination['expected']}") + if kind == 'load': + shortfalls += metrics['load']['issues'] + return ('inconclusive', shortfalls) if shortfalls else ('pass', []) + + +def repetition_verdict(idle, load): + """A failing idle baseline is inconclusive, never a failure of the load.""" + if idle['verdict'] != 'pass': + return 'inconclusive', [f"idle baseline {idle['verdict']}"] + idle['reasons'] + if load['verdict'] != 'pass': + return load['verdict'], load['reasons'] + return 'pass', [] + + +def combination_verdict(verdicts, required): + """Qualified only when every required repetition passed; values are never pooled.""" + if required < 1 or not verdicts: + return 'inconclusive' + if any(verdict == 'fail' for verdict in verdicts): + return 'failed' + if len(verdicts) >= required and all(verdict == 'pass' for verdict in verdicts): + return 'qualified' + return 'inconclusive' + + +def overall_verdict(combinations, plan, protocol, rehearsal): + """The run's verdict: qualified only for the full protocol, with every combination qualified.""" + verdicts = [entry['verdict'] for entry in combinations.values()] + full = (not rehearsal and set(plan['combinations']) >= set(protocol['combinations']) + and plan['repetitions'] >= protocol['repetitions'] + and plan['idle_s'] >= protocol['idle_s'] and plan['load_s'] >= protocol['load_s']) + if 'failed' in verdicts: + return 'failed' + if full and verdicts and all(verdict == 'qualified' for verdict in verdicts): + return 'qualified' + return 'inconclusive' + + +def check(name, passed, **detail): + """One assertion of a functional scenario. `passed` None means it could not be judged.""" + return {'check': name, 'passed': None if passed is None else bool(passed), **detail} + + +def scenario_verdict(checks, not_run=None): + """A scenario passes only when every check passed; a failed check fails it, and a check that + could not be judged leaves it inconclusive.""" + if not_run: + return 'not_run' + if any(c['passed'] is False for c in checks): + return 'fail' + if not checks or any(c['passed'] is None for c in checks): + return 'inconclusive' + return 'pass' + + +def suite_verdict(verdicts): + """A suite of scenarios: failed if any failed, passed only if every one that ran passed and + at least one ran; scenarios that cannot run on this platform are not counted.""" + ran = [v for v in verdicts if v != 'not_run'] + if 'fail' in ran: + return 'failed' + if ran and all(v == 'pass' for v in ran): + return 'passed' + return 'incomplete' diff --git a/scripts/qualify_host.py b/scripts/qualify_host.py new file mode 100644 index 0000000..d86851b --- /dev/null +++ b/scripts/qualify_host.py @@ -0,0 +1,913 @@ +"""CSRG-U6 qualification on the host: CodeSpace's gateway driven over its MCP stdio, DevGuard's +installed service and fixture authority observed, and the evidence written and sealed. + +Nothing here reads, prints or passes a credential. A gateway is given the credential file's path +and opens the file itself; DevGuard's journal is opened read-only and only the fields a check +needs are copied from it, never a permit's hash or a lease ID. Gateways and fixture authorities +run in a minimal environment (HOME, PATH, TMPDIR, LANG and CodeSpace's own settings) in their own +sessions, so a terminal's interrupt reaches only the harness, which then settles what it started. +""" +import hashlib +import json +import os +from pathlib import Path +import platform +import queue +import re +import sqlite3 +import subprocess +import threading +import time + +from qualify_analysis import TRANSPORT + +SUPPORT = Path.home() / 'Library/Application Support/DevGuard' +JOURNAL = SUPPORT / 'state/authority.sqlite' +SERVICE_LOG = Path.home() / 'Library/Logs/DevGuard/devguardd.log' +HOST_CONFIG = Path.home() / '.config/devguard/host.toml' +# The consumer the owner provisions for CodeSpace in the installed service's configuration. +CONSUMER = 'codespace' +GENERATION = 'codespace-g1' +CREDENTIAL_FILE = SUPPORT / 'credentials/codespace.secret' +MINIMAL_PATH = '/usr/bin:/bin:/usr/sbin:/sbin' +MCP_PROTOCOL = '2025-11-25' +CALL_TIMEOUT_S = 30.0 +# The code of a call that got no answer within its timeout; a lost connection is TRANSPORT. +NO_ANSWER = 'no_answer' +# Codes that are the transport's, not CodeSpace's answer. +TRANSPORT_CODES = (TRANSPORT, NO_ANSWER) + + +def default_socket(): + return Path(f'/private/tmp/devguard-{os.getuid()}/authority.sock') + + +def sha256(path): + digest = hashlib.sha256() + with open(path, 'rb') as handle: + for block in iter(lambda: handle.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() + + +def run_text(*args, timeout=30, env=None, cwd=None): + """A command's standard output, or None when it cannot run, fails or does not finish in time.""" + try: + result = subprocess.run([str(arg) for arg in args], capture_output=True, text=True, timeout=timeout, + env=env, cwd=cwd) + except (OSError, subprocess.TimeoutExpired): + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def run_json(*args, timeout=60): + text = run_text(*args, timeout=timeout) + try: + return json.loads(text) if text else None + except ValueError: + return None + + +def minimal_env(tmpdir, extra=None): + """The environment of every process the harness starts: no inherited variable reaches it.""" + env = {'HOME': str(Path.home()), 'PATH': MINIMAL_PATH, 'TMPDIR': str(tmpdir), 'LANG': 'C'} + env.update(extra or {}) + return env + + +# ---------------------------------------------------------------- evidence + +class Evidence: + """A new evidence directory. Raw files are written as the run goes; `seal` records every + file's SHA-256 in MANIFEST.json and makes the files read-only, and nothing is written after.""" + + def __init__(self, path): + self.path = Path(path).resolve() + if self.path.exists(): + raise SystemExit(f'qualify: {self.path} already exists') + self.path.mkdir(parents=True, mode=0o700) + self.lock = threading.Lock() + self.sealed = None + + def file(self, name): + path = self.path / name + path.parent.mkdir(parents=True, exist_ok=True) + return path + + def write_json(self, name, value): + with self.lock: + self.file(name).write_text(json.dumps(value, indent=2) + '\n') + + def append(self, name, row): + with self.lock: + with open(self.file(name), 'a') as handle: + handle.write(json.dumps(row) + '\n') + + def seal(self): + if self.sealed is None: + manifest = self.path / 'MANIFEST.json' + files = sorted(path for path in self.path.rglob('*') if path.is_file() and path != manifest) + manifest.write_text(json.dumps({ + 'schema': 'codespace-qualification-manifest/v1', 'sealed_unix': time.time(), + 'files': {str(path.relative_to(self.path)): sha256(path) for path in files}}, indent=1) + '\n') + for path in files + [manifest]: + os.chmod(path, 0o400) + self.sealed = sha256(manifest) + return self.sealed + + +def verify_manifest(directory): + """The files of a sealed directory that are missing, changed or not listed.""" + directory = Path(directory) + manifest = directory / 'MANIFEST.json' + listed = json.loads(manifest.read_text())['files'] + present = {str(path.relative_to(directory)) for path in directory.rglob('*') + if path.is_file() and path != manifest} + problems = [f'missing {name}' for name in sorted(set(listed) - present)] + problems += [f'not listed {name}' for name in sorted(present - set(listed))] + problems += [f'changed {name}' for name in sorted(set(listed) & present) + if sha256(directory / name) != listed[name]] + return problems + + +class Writer(threading.Thread): + """Appends JSON lines on its own thread, as DevGuard's measure.py does, so no sampler waits on + a disk write: a disk the load saturates delays the evidence, never a sample.""" + + def __init__(self): + super().__init__(daemon=True) + self.lines = queue.Queue() + self.files = {} + self.errors = [] + self.written = 0 + self.lag_max = 0.0 + self.report = None + self.start() + + def write(self, path, row): + self.lines.put((time.monotonic(), Path(path), json.dumps(row) + '\n')) + + def run(self): + ending = False + while not ending: + batch = [self.lines.get()] + while True: + try: + batch.append(self.lines.get_nowait()) + except queue.Empty: + break + ending = None in batch + batch = [item for item in batch if item is not None] + for _, path, line in batch: + try: + if path not in self.files: + path.parent.mkdir(parents=True, exist_ok=True) + self.files[path] = open(path, 'a') + self.files[path].write(line) + self.written += 1 + except OSError as error: + self.errors.append(f'{path.name}: {error}') + for path, handle in self.files.items(): + try: + handle.flush() + except OSError as error: + self.errors.append(f'{path.name}: {error}') + if batch: + self.lag_max = max(self.lag_max, time.monotonic() - batch[0][0]) + + def close(self, limit=120): + """Write every line handed over and close the files; a disk that does not take them within + `limit` seconds is reported, never waited for without end.""" + if self.report is None: + self.lines.put(None) + self.join(timeout=limit) + if self.is_alive(): + self.errors.append(f'the evidence writer did not finish within {limit} s') + for path, handle in list(self.files.items()): + try: + handle.close() + except OSError as error: + self.errors.append(f'{path.name}: {error}') + self.report = {'lines': self.written, 'lag_max_ms': round(self.lag_max * 1000, 3), + 'errors': self.errors[:20], 'error_count': len(self.errors)} + return self.report + + +class Sampler(threading.Thread): + """Calls `sample` every `period` seconds until stopped and hands each row to the writer; a + sample that raises is kept as an error row.""" + + def __init__(self, writer, path, period, sample): + super().__init__(daemon=True) + self.writer, self.path, self.period, self.sample = writer, path, period, sample + self.halt = threading.Event() + self.rows = [] + + def run(self): + next_at = time.monotonic() + while not self.halt.is_set(): + try: + row = {'unix': time.time(), **self.sample()} + except Exception as error: # every failed sample is kept as evidence + row = {'unix': time.time(), 'error': f'{type(error).__name__}: {error}'} + self.rows.append(row) + self.writer.write(self.path, row) + next_at += self.period + self.halt.wait(max(0.0, next_at - time.monotonic())) + + def stop(self): + self.halt.set() + self.join() + return self.rows + + +# ---------------------------------------------------------------- MCP + +class ToolError(Exception): + """A call that did not succeed: CodeSpace's error code and message, or the transport's + (`connection_lost`, `no_answer`, `rpc_`), with the round trip when an answer came.""" + + def __init__(self, code, message, body=None, latency_ms=None): + super().__init__(f'{code}: {message}') + self.code, self.message, self.body, self.latency_ms = code, message, body or {}, latency_ms + + @property + def transport(self): + return self.code in TRANSPORT_CODES or str(self.code).startswith('rpc_') + + +class McpClient: + """Newline-delimited JSON-RPC over a gateway's stdio, as rmcp speaks it. Calls may come from + several threads at once; each answer is matched by its id and timed when the reader thread + takes it off the pipe. A connection that ends fails every call in flight and every later one + with `connection_lost`.""" + + def __init__(self, process, record=None): + self.process = process + self.record = record + self.write_lock = threading.Lock() + self.lock = threading.Lock() + self.pending = {} + self.next_id = 0 + self.lost = None + self.malformed = 0 + self.reader = threading.Thread(target=self._read, daemon=True) + self.reader.start() + + def _read(self): + reason = 'the gateway closed its output' + try: + for line in self.process.stdout: + arrived = time.monotonic() + try: + message = json.loads(line) + except ValueError: + self.malformed += 1 + continue + if self.record: + self.record({'dir': 'in', 'mono': arrived, 'message': message}) + if not isinstance(message, dict): + self.malformed += 1 + elif 'method' in message: + self._answer_server(message) + else: + with self.lock: + waiter = self.pending.pop(message.get('id'), None) + if waiter: + waiter['message'], waiter['arrived'] = message, arrived + waiter['event'].set() + except (OSError, ValueError) as error: + reason = f'{type(error).__name__}: {error}' + with self.lock: + self.lost = reason + waiters = list(self.pending.values()) + self.pending.clear() + for waiter in waiters: + waiter['event'].set() + + def _answer_server(self, message): + if 'id' not in message: + return + if message['method'] == 'ping': + reply = {'jsonrpc': '2.0', 'id': message['id'], 'result': {}} + else: + reply = {'jsonrpc': '2.0', 'id': message['id'], 'error': {'code': -32601, 'message': 'method not found'}} + try: + self._send(reply) + except (OSError, ValueError): + pass + + def _send(self, message): + data = (json.dumps(message) + '\n').encode() + with self.write_lock: + self.process.stdin.write(data) + self.process.stdin.flush() + + def notify(self, method, params=None): + message = {'jsonrpc': '2.0', 'method': method} + if params is not None: + message['params'] = params + self._send(message) + if self.record: + self.record({'dir': 'out', 'mono': time.monotonic(), 'message': message}) + + def request(self, method, params, timeout=CALL_TIMEOUT_S): + """The answer's result and its round trip in milliseconds.""" + waiter = {'event': threading.Event(), 'message': None, 'arrived': None} + with self.lock: + if self.lost is not None: + raise ToolError(TRANSPORT, self.lost) + self.next_id += 1 + ident = self.next_id + self.pending[ident] = waiter + message = {'jsonrpc': '2.0', 'id': ident, 'method': method, 'params': params} + sent = time.monotonic() + try: + self._send(message) + except (OSError, ValueError) as error: + with self.lock: + self.pending.pop(ident, None) + raise ToolError(TRANSPORT, f'cannot write to the gateway: {error}') + if self.record: + self.record({'dir': 'out', 'mono': sent, 'message': message}) + if not waiter['event'].wait(timeout): + with self.lock: + self.pending.pop(ident, None) + raise ToolError(NO_ANSWER, f'no answer within {timeout} s') + if waiter['message'] is None: + raise ToolError(TRANSPORT, self.lost or 'the connection ended') + latency = round((waiter['arrived'] - sent) * 1000, 3) + answer = waiter['message'] + if 'error' in answer: + error = answer.get('error') or {} + raise ToolError(f"rpc_{error.get('code')}", str(error.get('message')), latency_ms=latency) + return answer.get('result') or {}, latency + + def call(self, name, arguments, timeout=CALL_TIMEOUT_S): + """A tool's payload and round trip in milliseconds; a tool error raises ToolError with + CodeSpace's code and the round trip.""" + result, latency = self.request('tools/call', {'name': name, 'arguments': arguments}, timeout) + text = ''.join(part.get('text', '') for part in result.get('content') or [] + if isinstance(part, dict) and part.get('type') == 'text') + if result.get('isError'): + try: + body = json.loads(text) + except ValueError: + body = None + body = body if isinstance(body, dict) else {} + raise ToolError(body.get('code') or 'TOOL_ERROR', body.get('message') or text, body, latency) + payload = result.get('structuredContent') + if payload is None: + try: + payload = json.loads(text) if text else {} + except ValueError: + payload = {'text': text} + return payload, latency + + +# ---------------------------------------------------------------- the gateway + +def launch_line(text): + """The governed execution a gateway log line reports, if it reports one: its attempt and + process IDs, whether it was launched, and the scope's root (the helper's PID).""" + if 'DevGuard managed launch' not in text and 'DevGuard preparation' not in text: + return None + attempt = re.search(r'\battempt_id=(cs-attempt-[0-9a-f]+)', text) + process = re.search(r'\bprocess_id=(\S+)', text) + if not (attempt and process): + return None + entry = {'attempt_id': attempt[1], 'process_id': process[1]} + if 'the command was not started' in text or 'DevGuard preparation' in text: + entry['launched'] = False + outcome = re.search(r'\boutcome=(\w+)', text) + entry['outcome'] = outcome[1] if outcome else None + else: + entry['launched'] = True + root = re.search(r'\bscope_root_pid=Some\((\d+)\)', text) + dispatch = re.search(r'\bdispatch=(\w+)', text) + entry['scope_root_pid'] = int(root[1]) if root else None + entry['dispatch'] = dispatch[1] if dispatch else None + return entry + + +def refusal_ids(message): + """The attempt and process IDs a governed refusal names.""" + match = re.search(r'attempt `(cs-attempt-[0-9a-f]+)` for process_id `([^`]+)`', message or '') + return (match[1], match[2]) if match else (None, None) + + +def registry(workspaces): + """A workspace registry: {id: (root, resources or None[, other settings])}, each root a + directory it creates.""" + entries = {} + for name, value in workspaces.items(): + root, resources = value[0], value[1] + Path(root).mkdir(parents=True, exist_ok=True) + entry = {'root': str(root), 'profile': 'workspace-write'} + if resources: + entry['resources'] = resources + if len(value) > 2: + entry.update(value[2]) + entries[name] = entry + return {'workspaces': entries} + + +def required(cpu_milli, memory_bytes, tasks, minimum=None): + request = {'cpu_milli': cpu_milli, 'memory_bytes': memory_bytes, 'tasks': tasks} + if minimum: + request['minimum'] = minimum + return {'participation': 'required', 'request': request} + + +def devguard_flags(mode, socket, credential_file, launch_helper=None, consumer=CONSUMER, generation=GENERATION): + """The gateway's DevGuard settings. The credential file is named, never read here.""" + flags = ['--devguard', mode, '--devguard-socket', str(socket), '--devguard-consumer', consumer, + '--devguard-generation', generation, '--devguard-credential-file', str(credential_file)] + if launch_helper: + flags += ['--devguard-launch-helper', str(launch_helper)] + return flags + + +def runner_flags(mode, runtime_bin=None, runner_dir=None, runner_socket=None): + if mode == 'in-process': + return ['--runner', 'in-process'] + if runner_socket: + return ['--runner', 'uds', '--runner-socket', str(runner_socket)] + return ['--runner', 'uds', '--runtime-bin', str(runtime_bin), '--runner-dir', str(runner_dir)] + + +class Gateway: + """A codespace-mcp process on stdio in its own session. Its log is kept as evidence, and the + governed executions it reports are indexed by process ID, every report kept, so that a + second launch of one execution shows. Every process it starts through `exec` or a resumed + approval is tracked, so `settle` can end them before the gateway goes.""" + + def __init__(self, binary, flags, env, log_path, record=None, cwd=None): + self.argv = [str(binary), *flags] + self.log_path = Path(log_path) + self.executions = {} + self.started = set() + self.lock = threading.Lock() + self.process = subprocess.Popen(self.argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, cwd=cwd, start_new_session=True) + self.pid = self.process.pid + self.log_reader = threading.Thread(target=self._log, daemon=True) + self.log_reader.start() + self.client = McpClient(self.process, record) + self.info = None + + def _log(self): + with open(self.log_path, 'ab') as log: + for line in self.process.stderr: + log.write(line) + log.flush() + entry = launch_line(line.decode('utf-8', 'replace')) + if entry: + with self.lock: + self.executions.setdefault(entry['process_id'], []).append(entry) + + def initialize(self, timeout=60): + result, latency = self.client.request('initialize', { + 'protocolVersion': MCP_PROTOCOL, 'capabilities': {}, + 'clientInfo': {'name': 'codespace-qualify', 'version': '1'}}, timeout) + self.client.notify('notifications/initialized') + self.info = {'result': result, 'latency_ms': latency} + return result + + def execution(self, process_id, limit=5.0): + """What the gateway's log last said of a governed execution, waiting briefly for it.""" + deadline = time.monotonic() + limit + while True: + with self.lock: + entries = self.executions.get(process_id) + if entries or time.monotonic() >= deadline: + return entries[-1] if entries else None + time.sleep(0.02) + + def launches(self): + """{process ID: number of launches the log reports} for every governed execution.""" + with self.lock: + return {pid: sum(1 for entry in entries if entry.get('launched')) + for pid, entries in self.executions.items()} + + def attempts(self): + """Every attempt the log names, launched or not.""" + with self.lock: + return sorted({entry['attempt_id'] for entries in self.executions.values() for entry in entries}) + + def call(self, name, arguments, timeout=CALL_TIMEOUT_S): + return self.client.call(name, arguments, timeout) + + def exec(self, workspace, command, tty=False, timeout=CALL_TIMEOUT_S): + """exec_command: the outcome (`started`, `refused` with CodeSpace's code, or `error` for the + transport's), the process ID, the dispatch status, the attempt a refusal names and the + round trip.""" + arguments = {'workspace_id': workspace, 'command': [str(part) for part in command]} + if tty: + arguments['tty'] = True + try: + payload, latency = self.client.call('exec_command', arguments, timeout) + except ToolError as error: + attempt, process = refusal_ids(error.message) + return {'outcome': 'error' if error.transport else 'refused', 'code': error.code, + 'message': error.message, 'latency_ms': error.latency_ms, 'attempt_id': attempt, + 'process_id': process, 'approval_id': error.body.get('approval_id')} + process_id = payload.get('process_id') + with self.lock: + self.started.add(process_id) + return {'outcome': 'started', 'process_id': process_id, 'dispatch': payload.get('dispatch_status'), + 'latency_ms': latency} + + def resume(self, approval_id, timeout=CALL_TIMEOUT_S): + """operation_resume of a granted approval; the execution it returns is tracked.""" + payload, latency = self.client.call('operation_resume', {'approval_id': approval_id}, timeout) + process_id = (payload.get('exec_command') or {}).get('process_id') + if process_id: + with self.lock: + self.started.add(process_id) + return payload, latency + + def status(self, process_id, timeout=CALL_TIMEOUT_S): + return self.client.call('process_status', {'process_id': process_id}, timeout) + + def wait_exited(self, process_id, limit=30.0, period=0.05): + """The status once the process has exited, or None after `limit` seconds.""" + deadline = time.monotonic() + limit + while True: + payload, _ = self.status(process_id) + if payload.get('state') == 'exited': + return payload + if time.monotonic() >= deadline: + return None + time.sleep(period) + + def read_until(self, process_id, wanted=None, limit=10.0, period=0.02, cursor=0, timeout=CALL_TIMEOUT_S): + """Read output from `cursor` until `wanted` appears in it (None: until the output ends), + the output ends or `limit` seconds pass: (text, next cursor, eof, found). A terminal's + line ends are read as plain newlines.""" + deadline, text, eof = time.monotonic() + limit, '', False + while True: + payload, _ = self.client.call('read_process', {'process_id': process_id, 'cursor': cursor}, timeout) + text += (payload.get('chunk') or '').replace('\r\n', '\n') + cursor, eof = payload.get('cursor', cursor), bool(payload.get('eof')) + found = wanted is not None and wanted in text + if found or eof or time.monotonic() >= deadline: + return text, cursor, eof, found + time.sleep(period) + + def terminate(self, process_id, timeout=CALL_TIMEOUT_S): + _, latency = self.client.call('terminate_process', {'process_id': process_id}, timeout) + return latency + + def settle(self, limit=30.0): + """End every process this harness started that still runs, and wait for each to exit.""" + with self.lock: + started = sorted(pid for pid in self.started if pid) + left = [] + for process_id in started: + try: + payload, _ = self.status(process_id, timeout=10) + if payload.get('state') != 'exited': + self.terminate(process_id, timeout=10) + if self.wait_exited(process_id, limit=limit) is None: + left.append(process_id) + except ToolError as error: + if error.transport: + left.append(process_id) + return left + + def close(self, limit=60): + """Close the gateway's input, as a client that goes away does, and wait for it to exit; one + that does not exit in time is stopped. Its log is complete afterwards.""" + try: + self.process.stdin.close() + except OSError: + pass + stopped = None + try: + self.process.wait(timeout=limit) + except subprocess.TimeoutExpired: + stopped = 'SIGTERM' + self.process.terminate() + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + stopped = 'SIGKILL' + self.process.kill() + self.process.wait() + self.log_reader.join(timeout=10) + return {'returncode': self.process.returncode, 'stopped': stopped, + 'malformed_lines': self.client.malformed} + + +# ---------------------------------------------------------------- processes and descriptors + +def group_alive(pgid): + """Whether any process is still in the process group, a zombie included.""" + try: + os.killpg(pgid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def wait_group_gone(pgid, limit=30.0, period=0.01): + """Seconds until the group was observed empty, or None after `limit`.""" + began = time.monotonic() + while group_alive(pgid): + if time.monotonic() - began >= limit: + return None + time.sleep(period) + return time.monotonic() - began + + +def descriptors(pid): + """The numbered descriptors a process holds, from lsof, or None when they cannot be read.""" + text = run_text('lsof', '-nP', '-a', '-p', str(pid), '-d', '0-65535', '-F', 'f', timeout=30) + if text is None: + return None + return sorted(int(line[1:]) for line in text.splitlines() if line.startswith('f') and line[1:].isdigit()) + + +def children(pid): + text = run_text('pgrep', '-P', str(pid), timeout=10) or '' + return [int(value) for value in text.split()] + + +def process_footprint(pid): + """RSS, threads and CPU share of one process, from ps; None when it is gone.""" + line = run_text('ps', '-o', 'rss=,%cpu=', '-p', str(pid), timeout=10) + if not line: + return None + fields = line.split() + if platform.system() == 'Darwin': + listing = run_text('ps', '-M', '-p', str(pid), timeout=10) + threads = max(0, len(listing.splitlines()) - 1) if listing else None + else: + text = run_text('ps', '-o', 'nlwp=', '-p', str(pid), timeout=10) + threads = int(text) if text else None + return {'pid': pid, 'rss_bytes': int(fields[0]) * 1024, 'cpu_percent': float(fields[1]), 'threads': threads} + + +def host_sample(): + """Memory pressure, swap and load, as DevGuard's measure.py samples them.""" + row = {'load_average': list(os.getloadavg())} + if platform.system() == 'Darwin': + swap = run_text('/usr/sbin/sysctl', '-n', 'vm.swapusage') or '' + used = re.search(r'used = ([\d.]+)M', swap) + row['pressure_level'] = run_text('/usr/sbin/sysctl', '-n', 'kern.memorystatus_vm_pressure_level') + row['swap_used_mib'] = float(used[1]) if used else None + return row + + +def host_facts(): + """The host's identity and condition for the candidate record.""" + def sysctl(name): + return run_text('/usr/sbin/sysctl', '-n', name) + facts = {'platform': platform.platform(), 'machine': platform.machine(), 'kernel': platform.release(), + 'python': platform.python_version()} + if platform.system() == 'Darwin': + facts.update({'macos': platform.mac_ver()[0], 'build': run_text('/usr/bin/sw_vers', '-buildVersion'), + 'model': sysctl('hw.model'), 'cpu': sysctl('machdep.cpu.brand_string'), + 'logical_cpus': sysctl('hw.logicalcpu'), 'memory_bytes': sysctl('hw.memsize'), + 'boot_session': sysctl('kern.bootsessionuuid'), + 'power': (run_text('/usr/bin/pmset', '-g', 'batt') or '').splitlines()[:1]}) + else: + facts.update({'logical_cpus': str(os.cpu_count())}) + return facts + + +# ---------------------------------------------------------------- the installed DevGuard service + +class Release: + """An installed DevGuard release as the service runs it: its own directory, or its recovery + copy (DevGuard's measure.py finds it the same way).""" + + def __init__(self, release_id, support=SUPPORT): + self.id = release_id + directory = support / 'releases' / release_id + for candidate in (directory, support / 'recovery' / release_id): + status = run_json(candidate / 'bin/devguardd', 'status') + executable = Path(((status or {}).get('running') or {}).get('executable') or '') + if executable.name == 'devguardd' and executable.parent.parent.name == release_id: + directory = executable.parent.parent + break + self.directory = directory + self.manifest = directory / 'MANIFEST.json' + self.devguardd = directory / 'bin/devguardd' + self.devguard = directory / 'bin/devguard' + self.helper = directory / 'bin/devguard-launch' + + def state(self): + """Whether the service runs this release, and its process.""" + manifest = json.loads(self.manifest.read_text()) + status = run_json(self.devguardd, 'status') or {} + running = status.get('running') or {} + current = (status.get('selection') or {}).get('current') + return {'healthy': bool(status.get('healthy')), 'current': current, + 'running_sha256': running.get('sha256'), 'service_pid': running.get('service_pid'), + 'runs_release': bool(status.get('healthy') and current == self.id + and running.get('sha256') == manifest['artifacts']['devguardd']['sha256'])} + + def doctor(self): + report = run_json(self.devguard, 'doctor', '--require', 'admission,registration,macos-cooperative') or {} + service = ((report.get('checks') or {}).get('service') or {}).get('detail') or {} + return {'satisfied': report.get('satisfied'), 'requirements': report.get('requirements'), + 'protocol': service.get('protocol'), 'capabilities': service.get('capabilities'), + 'configuration_fingerprint': (service.get('status') or {}).get('configuration_fingerprint'), + 'authority_pid': (service.get('authority') or {}).get('pid')} + + def artifacts(self): + """The manifest's artifacts and whether each installed binary still matches it.""" + manifest = json.loads(self.manifest.read_text()) + found = {} + for name, entry in sorted(manifest.get('artifacts', {}).items()): + path = self.directory / 'bin' / name + found[name] = {'sha256': entry.get('sha256'), + 'matches': path.is_file() and sha256(path) == entry.get('sha256')} + return {'manifest_sha256': sha256(self.manifest), 'source': manifest.get('source'), 'artifacts': found} + + +def service_events(path=SERVICE_LOG, offset=0, kinds=None): + """The service's JSON receipts from `offset` on, optionally only those of the given kinds.""" + events = [] + try: + with open(path, 'rb') as log: + log.seek(offset) + for raw in log: + start = raw.find(b'{') + if start < 0: + continue + try: + event = json.loads(raw[start:]) + except ValueError: + continue + if isinstance(event, dict) and (kinds is None or event.get('event') in kinds): + events.append(event) + except OSError: + return [] + return events + + +def log_size(path=SERVICE_LOG): + try: + return Path(path).stat().st_size + except OSError: + return None + + +def pressure_state(path=SERVICE_LOG, window=4 << 20): + """The service's latest pressure state, from its receipts.""" + size = log_size(path) or 0 + events = service_events(path, max(0, size - window), ('pressure', 'pressure_control_lag')) + return events[-1].get('state') if events else None + + +def work_capacity(path=SERVICE_LOG): + """The work capacity the service reported when it last started.""" + events = service_events(path, 0, ('native_host',)) + return events[-1].get('work_capacity') if events else None + + +def journal_query(query, parameters=(), database=None): + connection = sqlite3.connect(f'file:{database or JOURNAL}?mode=ro', uri=True, timeout=5) + try: + return connection.execute(query, parameters).fetchall() + finally: + connection.close() + + +def attempt_view(record): + """What the checks need from one attempt record; never its permit hash or lease ID.""" + reservation = record.get('reservation') or {} + plan = record.get('plan') or {} + applied = record.get('applied') or {} + scope = record.get('scope') or {} + return {'phase': record.get('phase'), 'reserved': reservation.get('quantities'), + 'plan': {name: (plan.get(name) or {}).get('level') for name in ('cpu', 'memory', 'pids')} if plan else None, + 'applied': {name: applied.get(name) for name in ('cpu', 'memory', 'pids')} if applied else None, + 'applied_quantities': applied.get('quantities'), 'scope_kind': scope.get('kind'), + 'scope_root_pid': (scope.get('root') or {}).get('pid'), 'denial': record.get('denial'), + 'tracking_lost': record.get('tracking_lost'), 'release_reason': record.get('release_reason'), + 'owner_pid': ((record.get('owner') or {}).get('process') or {}).get('pid'), + 'policy_revision': record.get('policy_revision')} + + +def journal_attempts(attempts=None, database=None, consumer=CONSUMER): + """A consumer's attempts in DevGuard's journal, read-only: all of them, or the given IDs.""" + query = ('SELECT attempt, generation, charged, launch_hash IS NOT NULL, record FROM attempts ' + 'WHERE consumer = ?') + rows = [] + if attempts is None: + rows = journal_query(query, (consumer,), database) + else: + ids = sorted(set(attempts)) + for start in range(0, len(ids), 500): + chunk = ids[start:start + 500] + rows += journal_query(f"{query} AND attempt IN ({','.join('?' * len(chunk))})", (consumer, *chunk), + database) + found = {} + for attempt, generation, charged, committed, record in rows: + view = attempt_view(json.loads(record)) + # A committed grant later released as never started (no helper existed) did not run. + launched = bool(committed) and not (view['phase'] == 'released' + and view['release_reason'] == 'no_helper_created') + found[attempt] = dict(view, generation=generation, charged=bool(charged), launched=launched) + return found + + +def journal_charged(database=None): + """What still holds resources, by consumer.""" + attempts = journal_query('SELECT consumer, COUNT(*) FROM attempts WHERE charged = 1 GROUP BY consumer', + (), database) + leases = journal_query('SELECT COUNT(*) FROM leases WHERE charged = 1', (), database)[0][0] + return {'attempts': {consumer: count for consumer, count in attempts}, 'leases': leases} + + +def journal_instances(consumer=CONSUMER, database=None): + rows = journal_query('SELECT generation, instance, state, identity FROM instances WHERE consumer = ?', + (consumer,), database) + found = [] + for generation, instance, state, identity in rows: + process = (json.loads(identity) or {}).get('process') or {} + found.append({'generation': generation, 'instance': instance, 'state': state, 'pid': process.get('pid')}) + return found + + +def settle_attempts(ids, database=None, limit=60.0, consumer=CONSUMER): + """Wait until none of `ids` is charged: their views and those still charged.""" + deadline = time.monotonic() + limit + while True: + found = journal_attempts(ids, database, consumer) + charged = sorted(attempt for attempt, view in found.items() if view['charged']) + if not charged or time.monotonic() >= deadline: + return found, charged + time.sleep(0.5) + + +# ---------------------------------------------------------------- DevGuard's fixture authority + +class FixtureAuthority: + """DevGuard's fixture authority at CodeSpace's pin (crates/devguard/examples/fixture_authority.rs) + in its own process and session, under a private base directory. It provisions a `codespace` + consumer itself; its credential stays in its file, whose path the gateway is given.""" + + def __init__(self, binary, base, env, log_path, limit=2, timeout=60): + self.base = Path(base) + self.base.mkdir(parents=True, mode=0o700, exist_ok=True) + self.log = open(log_path, 'ab') + self.process = subprocess.Popen([str(binary), str(base), str(limit)], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=self.log, env=env, text=True, + start_new_session=True) + self.lines = queue.Queue() + threading.Thread(target=self._read, daemon=True).start() + self.settings = self._answer(timeout) + if self.settings is None: + self.kill() + raise RuntimeError('the fixture authority did not start') + + def _read(self): + for line in self.process.stdout: + self.lines.put(line) + self.lines.put(None) + + def _answer(self, timeout): + try: + line = self.lines.get(timeout=timeout) + except queue.Empty: + return None + return json.loads(line) if line else None + + def command(self, line, timeout=60): + self.process.stdin.write(line + '\n') + self.process.stdin.flush() + return self._answer(timeout) + + def charged(self): + return self.command('charged') + + def instances(self): + return self.command('instances') + + def journal(self): + """The fixture's journal, which the harness reads read-only as the installed service's.""" + return next(iter(sorted(self.base.rglob('authority.sqlite'))), None) + + def kill(self): + if self.process.poll() is None: + self.process.kill() + self.process.wait() + + def stop(self, limit=30): + try: + self.process.stdin.close() + except OSError: + pass + try: + self.process.wait(timeout=limit) + except subprocess.TimeoutExpired: + self.kill() + self.log.close() + return self.process.returncode diff --git a/scripts/tests/test_qualify_devguard.py b/scripts/tests/test_qualify_devguard.py new file mode 100644 index 0000000..5287181 --- /dev/null +++ b/scripts/tests/test_qualify_devguard.py @@ -0,0 +1,724 @@ +"""The CSRG-U6 harness without a host: its analysis, evidence, journal reads, MCP client and +verdict recomputation, on synthetic inputs (docs/cs-rg-qualification.md).""" +import contextlib +import importlib.util +import io +import json +import math +import os +from pathlib import Path +import sqlite3 +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest + +SCRIPTS = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS)) +import qualify_analysis as analysis # noqa: E402 +import qualify_host as host # noqa: E402 +spec = importlib.util.spec_from_file_location('qualify', SCRIPTS / 'qualify-devguard.py') +qualify = importlib.util.module_from_spec(spec) +spec.loader.exec_module(qualify) + +GIT_ENV = dict(os.environ, GIT_AUTHOR_NAME='t', GIT_AUTHOR_EMAIL='t@example.invalid', + GIT_COMMITTER_NAME='t', GIT_COMMITTER_EMAIL='t@example.invalid', + GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM='1') +RESERVATION = {'cpu_milli': 150, 'memory_bytes': 536870912, 'tasks': 40} + + +def answered(count, latency=5.0, state='running'): + return [{'slot': n, 'outcome': 'answered', 'latency_ms': latency, 'state': state} for n in range(count)] + + +def terminated(count, ack=20.0, released=300.0): + rows = [{'slot': n, 'outcome': 'acknowledged', 'ack_ms': ack, 'termination': 'terminated', + 'exited_ms': ack + 10, 'scope_exit_ms': ack + 15, 'attempt_id': f'cs-attempt-{n:02x}'} + for n in range(count)] + releases = [{'slot': n, 'attempt_id': f'cs-attempt-{n:02x}', 'released_ms': released, 'phase': 'released', + 'release_reason': 'scope_terminated'} for n in range(count)] + return rows, releases + + +def echoed(count, latency=3.0): + return [{'slot': n, 'outcome': 'echoed', 'latency_ms': latency, 'eof': False} for n in range(count)] + + +def refused(count, code='RESOURCE_UNAVAILABLE', latency=40.0): + return [{'slot': n, 'outcome': 'refused', 'code': code, 'latency_ms': latency} for n in range(count)] + + +def admitted(count, latency=60.0): + return [{'slot': n, 'outcome': 'admitted', 'latency_ms': latency, 'attempt_id': f'cs-attempt-a{n}'} + for n in range(count)] + + +class AnalysisTests(unittest.TestCase): + def test_a_missing_sample_ranks_above_every_value(self): + self.assertEqual(analysis.nearest_rank([10, 20], 0.99, missing=1), math.inf) + self.assertEqual(analysis.number(math.inf), 'missing') + self.assertEqual(analysis.nearest_rank([30, 10, 20], 0.5), 20) + self.assertIsNone(analysis.nearest_rank([], 0.99)) + + def test_status_target_is_the_99th_percentile(self): + fast = answered(99, 100.0) + answered(1, 600.0) + self.assertTrue(analysis.status_metrics(fast, 100)['passed']) + slow = answered(98, 100.0) + answered(2, 600.0) + self.assertFalse(analysis.status_metrics(slow, 100)['passed']) + + def test_an_answer_about_an_ended_target_is_missing(self): + metrics = analysis.status_metrics(answered(99) + answered(1, state='exited'), 100) + self.assertEqual(metrics['not_running'], 1) + self.assertEqual(metrics['p99_ms'], 5.0) + metrics = analysis.status_metrics(answered(98) + answered(2, state='exited'), 100) + self.assertEqual(metrics['p99_ms'], 'missing') + self.assertFalse(metrics['passed']) + + def test_a_lost_connection_fails_whatever_the_latency(self): + rows = answered(200) + [{'slot': 200, 'outcome': 'error', 'code': analysis.TRANSPORT}] + self.assertFalse(analysis.status_metrics(rows, 201)['passed']) + self.assertTrue(analysis.connection_lost([], rows)) + + def test_terminations(self): + rows, releases = terminated(10) + joined = analysis.with_releases(rows, releases) + self.assertTrue(analysis.termination_metrics(joined, 10)['passed']) + ineffective = joined[:9] + [dict(joined[9], termination='exited')] + self.assertFalse(analysis.termination_metrics(ineffective, 10)['passed']) + unreleased = analysis.with_releases(rows, releases[:9]) + self.assertFalse(analysis.termination_metrics(unreleased, 10)['passed']) + reported = analysis.termination_metrics(unreleased, 10, release_expected=False) + self.assertTrue(reported['passed']) + self.assertEqual(reported['unreleased'], 1) + refused_for_capacity = joined + [{'slot': 10, 'outcome': 'not_started', 'code': 'RESOURCE_UNAVAILABLE'}] + metrics = analysis.termination_metrics(refused_for_capacity, 11) + self.assertEqual((metrics['samples'], metrics['not_started']), (10, 1)) + + def test_control_metrics_counts_both_status_targets(self): + rows, releases = terminated(3) + metrics = analysis.control_metrics( + {'status_pipe': answered(60), 'status_pty': answered(60), 'terminations': rows, 'releases': releases, + 'io': echoed(12), 'refusals': refused(2)}, 60.4, qualify.PERIODS) + self.assertEqual((metrics['status']['samples'], metrics['status']['expected']), (120, 120)) + self.assertEqual(metrics['termination']['expected'], 3) + self.assertEqual(metrics['termination']['release_p99_ms'], 300.0) + self.assertEqual(metrics['io']['expected'], 12) + self.assertEqual(metrics['refusal']['codes'], {'RESOURCE_UNAVAILABLE': 2}) + self.assertFalse(metrics['connection_lost']) + + def test_io_and_refusals(self): + self.assertFalse(analysis.io_metrics(echoed(4) + [{'slot': 4, 'outcome': 'not_echoed'}], 5)['passed']) + self.assertTrue(analysis.io_metrics(echoed(5), 5)['passed']) + self.assertIsNone(analysis.refusal_metrics([])['passed']) + metrics = analysis.refusal_metrics(refused(2) + admitted(1)) + self.assertEqual((metrics['refused'], metrics['admitted'], metrics['passed']), (2, 1, True)) + + def interval(self, **changes): + rows, releases = terminated(30) + metrics = analysis.control_metrics( + {'status_pipe': answered(600), 'status_pty': answered(600), 'terminations': rows, 'releases': releases, + 'io': echoed(120), 'refusals': refused(20)}, 600.2, qualify.PERIODS) + metrics.update(service={'rows': 10, 'failure': None}, journal={'still_charged': [], 'duplicate_launches': 0}) + metrics['termination']['status_target_effective'] = True + for key, value in changes.items(): + metrics[key] = value + return metrics + + def test_interval_verdicts(self): + self.assertEqual(analysis.interval_verdict('idle', [], self.interval()), ('pass', [])) + self.assertEqual(analysis.interval_verdict('idle', ['the journal could not be read'], self.interval())[0], + 'inconclusive') + failing = self.interval(service={'failure': 'the release restarted or was unhealthy during the interval'}) + self.assertEqual(analysis.interval_verdict('idle', [], failing)[0], 'fail') + duplicate = self.interval(journal={'duplicate_launches': 1}) + self.assertIn('an execution was launched more than once', analysis.interval_verdict('idle', [], duplicate)[1]) + short = self.interval() + short['status'] = analysis.status_metrics(answered(1000), 1200) + self.assertEqual(analysis.interval_verdict('idle', [], short)[0], 'inconclusive') + target = self.interval() + target['termination'] = dict(target['termination'], status_target_effective=False) + self.assertEqual(analysis.interval_verdict('idle', [], target)[0], 'fail') + + def test_a_failing_baseline_is_inconclusive_and_repetitions_are_not_pooled(self): + verdict, _ = analysis.repetition_verdict({'verdict': 'fail', 'reasons': ['x']}, {'verdict': 'pass', 'reasons': []}) + self.assertEqual(verdict, 'inconclusive') + self.assertEqual(analysis.combination_verdict(['pass', 'pass', 'pass'], 3), 'qualified') + self.assertEqual(analysis.combination_verdict(['pass', 'pass'], 3), 'inconclusive') + self.assertEqual(analysis.combination_verdict(['pass', 'fail', 'pass'], 3), 'failed') + plan = dict(qualify.CONTROL_PROTOCOL) + qualified = {mode: {'verdict': 'qualified'} for mode in plan['combinations']} + self.assertEqual(analysis.overall_verdict(qualified, plan, qualify.CONTROL_PROTOCOL, False), 'qualified') + self.assertEqual(analysis.overall_verdict(qualified, plan, qualify.CONTROL_PROTOCOL, True), 'inconclusive') + one = {'uds': {'verdict': 'qualified'}} + self.assertEqual(analysis.overall_verdict(one, dict(plan, combinations=['uds']), qualify.CONTROL_PROTOCOL, + False), 'inconclusive') + + def test_scenarios_and_suites(self): + check = analysis.check + self.assertEqual(analysis.scenario_verdict([check('a', True), check('b', True)]), 'pass') + self.assertEqual(analysis.scenario_verdict([check('a', True), check('b', None)]), 'inconclusive') + self.assertEqual(analysis.scenario_verdict([check('a', True), check('b', False)]), 'fail') + self.assertEqual(analysis.scenario_verdict([]), 'inconclusive') + self.assertEqual(analysis.suite_verdict(['pass', 'not_run']), 'passed') + self.assertEqual(analysis.suite_verdict(['pass', 'inconclusive']), 'incomplete') + self.assertEqual(analysis.suite_verdict(['pass', 'fail']), 'failed') + self.assertEqual(analysis.suite_verdict([]), 'incomplete') + + def test_load_shares_and_failures(self): + self.assertEqual(analysis.covered([(0, 4), (2, 6), (8, 9)], 0, 10), 0.7) + runs = [{'consumer': 'cargo', 'result': 'completed', 'exit_code': 0, 'running_from': 0, 'ended': 8}, + {'consumer': 'tests', 'result': 'uncertain', 'running_from': 1, 'ended': 2}] + metrics = analysis.load_metrics(runs, 0, 10, ('cargo', 'tests', 'idle')) + self.assertEqual(metrics['applied_fraction'], 0.8) + self.assertEqual(metrics['failures'], ['1 load runs are uncertain']) + self.assertIn('idle: no successful run', metrics['issues']) + self.assertIn('tests: ran for 10% of the interval', metrics['issues']) + + def test_service_findings(self): + good = [{'runs_release': True, 'current': 'r1', 'service_pid': 7}] + self.assertEqual(analysis.service_findings(good, 'r1', 7), ([], None)) + self.assertEqual(analysis.service_findings(good, 'r1', 8)[1], + 'the release restarted or was unhealthy during the interval') + self.assertEqual(analysis.service_findings([{'current': 'r2'}], 'r1', 7)[0], + ['another release was selected during the interval']) + self.assertEqual(analysis.service_findings([], 'r1', 7)[0], ['the service was not observed']) + + +LAUNCHED = ('2026-10-12T16:00:00.000001Z INFO codespace_server::devguard: DevGuard managed launch ' + 'attempt_id=cs-attempt-1f2e process_id=proc-7c1d dispatch=Confirmed phases=[Prepared, Committed] ' + 'applied=Some(Applied) scope_root_pid=Some(4242)') +NOT_STARTED = ('2026-10-12T16:00:01.000001Z INFO codespace_server::devguard: DevGuard managed launch; the ' + 'command was not started attempt_id=cs-attempt-3a4b process_id=proc-9e8f outcome=ExecFailed attempt=x') + + +class HostTests(unittest.TestCase): + def setUp(self): + self.scratch = tempfile.TemporaryDirectory() + self.dir = Path(self.scratch.name) + + def tearDown(self): + for path in self.dir.rglob('*'): + os.chmod(path, 0o700 if path.is_dir() else 0o600) + self.scratch.cleanup() + + def test_launch_lines(self): + self.assertEqual(host.launch_line(LAUNCHED), {'attempt_id': 'cs-attempt-1f2e', 'process_id': 'proc-7c1d', + 'launched': True, 'scope_root_pid': 4242, + 'dispatch': 'Confirmed'}) + self.assertEqual(host.launch_line(NOT_STARTED), {'attempt_id': 'cs-attempt-3a4b', 'process_id': 'proc-9e8f', + 'launched': False, 'outcome': 'ExecFailed'}) + self.assertIsNone(host.launch_line('INFO codespace_server: started')) + + def test_refusal_ids(self): + message = 'attempt `cs-attempt-00ff` for process_id `proc-1`: denied; nothing was started' + self.assertEqual(host.refusal_ids(message), ('cs-attempt-00ff', 'proc-1')) + self.assertEqual(host.refusal_ids('busy'), (None, None)) + + def test_registry_and_flags(self): + spaces = {'gov': (self.dir / 'ws/gov', host.required(25, 16 << 20, 2)), + 'off': (self.dir / 'ws/off', None), + 'confirm': (self.dir / 'ws/confirm', host.required(25, 16 << 20, 2), {'approvals': 'confirm'})} + registry = host.registry(spaces)['workspaces'] + self.assertTrue((self.dir / 'ws/gov').is_dir()) + self.assertEqual(registry['gov']['resources'], {'participation': 'required', 'request': { + 'cpu_milli': 25, 'memory_bytes': 16 << 20, 'tasks': 2}}) + self.assertNotIn('resources', registry['off']) + self.assertEqual(registry['confirm']['approvals'], 'confirm') + flags = host.devguard_flags('register', '/s.sock', '/c.secret', '/bin/helper') + self.assertEqual(flags[flags.index('--devguard-credential-file') + 1], '/c.secret') + self.assertEqual(flags[-2:], ['--devguard-launch-helper', '/bin/helper']) + self.assertEqual(host.runner_flags('in-process'), ['--runner', 'in-process']) + self.assertIn('--runner-socket', host.runner_flags('uds', runner_socket='/w.sock')) + + def test_sealed_evidence_shows_every_change(self): + evidence = host.Evidence(self.dir / 'run') + evidence.write_json('a/report.json', {'verdict': 'pass'}) + evidence.append('a/rows.jsonl', {'slot': 0}) + manifest = evidence.seal() + self.assertEqual(evidence.seal(), manifest) + self.assertEqual(host.verify_manifest(evidence.path), []) + os.chmod(evidence.path / 'a/rows.jsonl', 0o600) + (evidence.path / 'a/rows.jsonl').write_text('{"slot": 1}\n') + os.chmod(evidence.path / 'a', 0o700) + (evidence.path / 'a/extra.json').write_text('{}\n') + (evidence.path / 'a/report.json').unlink() + self.assertEqual(host.verify_manifest(evidence.path), + ['missing a/report.json', 'not listed a/extra.json', 'changed a/rows.jsonl']) + with self.assertRaises(SystemExit): + host.Evidence(self.dir / 'run') + + def test_writer(self): + writer = host.Writer() + for n in range(100): + writer.write(self.dir / 'w/rows.jsonl', {'n': n}) + report = writer.close() + self.assertEqual((report['lines'], report['error_count']), (100, 0)) + self.assertEqual(writer.close(), report) + lines = (self.dir / 'w/rows.jsonl').read_text().splitlines() + self.assertEqual([json.loads(line)['n'] for line in lines], list(range(100))) + + def journal(self): + database = self.dir / 'authority.sqlite' + connection = sqlite3.connect(database) + connection.executescript(""" + CREATE TABLE attempts (attempt TEXT, consumer TEXT, generation TEXT, charged INTEGER, launch_hash TEXT, + record TEXT); + CREATE TABLE leases (lease TEXT, charged INTEGER); + CREATE TABLE instances (consumer TEXT, generation TEXT, instance TEXT, state TEXT, identity TEXT); + """) + + def record(phase, reason=None, root=None): + return json.dumps({ + 'phase': phase, 'reservation': {'quantities': {'cpu_milli': 25, 'memory_bytes': 1, 'tasks': 2}}, + 'applied': {'cpu': 'applied', 'memory': 'applied', 'pids': 'applied', + 'quantities': {'cpu_milli': 25, 'memory_bytes': 1, 'tasks': 2}}, + 'scope': {'kind': 'process_group', 'root': {'pid': root}}, 'tracking_lost': False, + 'release_reason': reason, 'owner': {'process': {'pid': 7}}, 'permit_sha256': 'f' * 64, + 'lease_id': 'lease-secret'}) + rows = [('cs-attempt-1', 'codespace', 'g1', 0, 'h', record('released', 'scope_terminated', 99)), + ('cs-attempt-2', 'codespace', 'g1', 1, 'h', record('suspect', None, 98)), + ('cs-attempt-3', 'codespace', 'g1', 0, 'h', record('released', 'no_helper_created')), + ('cs-attempt-4', 'codespace', 'g1', 0, None, record('denied')), + ('cs-attempt-5', 'dev-cli', 'g0', 1, 'h', record('committed', None, 97))] + connection.executemany('INSERT INTO attempts VALUES (?, ?, ?, ?, ?, ?)', rows) + connection.execute("INSERT INTO leases VALUES ('lease-secret', 1)") + connection.execute("INSERT INTO instances VALUES ('codespace', 'g1', 'i1', 'active', ?)", + (json.dumps({'process': {'pid': 7}}),)) + connection.commit() + connection.close() + return database + + def test_the_journal_is_read_without_permits_or_leases(self): + database = self.journal() + found = host.journal_attempts(None, database) + self.assertEqual(sorted(found), ['cs-attempt-1', 'cs-attempt-2', 'cs-attempt-3', 'cs-attempt-4']) + self.assertNotIn('secret', json.dumps(found)) + self.assertNotIn('f' * 64, json.dumps(found)) + self.assertTrue(found['cs-attempt-1']['launched']) + self.assertFalse(found['cs-attempt-3']['launched']) + self.assertFalse(found['cs-attempt-4']['launched']) + self.assertEqual(found['cs-attempt-1']['scope_root_pid'], 99) + self.assertEqual(sorted(host.journal_attempts(['cs-attempt-2', 'cs-attempt-5'], database)), ['cs-attempt-2']) + self.assertEqual(host.journal_charged(database), {'attempts': {'codespace': 1, 'dev-cli': 1}, 'leases': 1}) + self.assertEqual(host.journal_instances(database=database), + [{'generation': 'g1', 'instance': 'i1', 'state': 'active', 'pid': 7}]) + found, charged = host.settle_attempts(['cs-attempt-1', 'cs-attempt-2'], database, limit=0) + self.assertEqual(charged, ['cs-attempt-2']) + + +FAKE_SERVER = textwrap.dedent(""" + import json, sys, time + def send(message): + sys.stdout.write(json.dumps(message) + '\\n') + sys.stdout.flush() + def text(value, error=False): + return {'content': [{'type': 'text', 'text': json.dumps(value)}], 'isError': error} + for line in sys.stdin: + message = json.loads(line) + if 'method' not in message: + continue + name = message.get('params', {}).get('name') + if message['method'] == 'initialize': + send({'jsonrpc': '2.0', 'id': message['id'], 'result': {'protocolVersion': '2025-11-25'}}) + elif name == 'echo': + send({'jsonrpc': '2.0', 'id': message['id'], + 'result': dict(text({}), structuredContent=message['params']['arguments'])}) + elif name == 'exec_command' and message['params']['arguments']['command'] == ['busy']: + send({'jsonrpc': '2.0', 'id': message['id'], 'result': text( + {'code': 'WORKSPACE_BUSY', 'message': 'attempt `cs-attempt-0a` for process_id `proc-2`: busy', + 'approval_id': 'appr-1'}, error=True)}) + elif name == 'ping-me': + send({'jsonrpc': '2.0', 'id': 'server-1', 'method': 'ping'}) + reply = json.loads(sys.stdin.readline()) + send({'jsonrpc': '2.0', 'id': message['id'], 'result': text({'pinged': reply.get('result') == {}})}) + elif name == 'silent': + pass + elif name == 'exit': + sys.exit(0) +""") + + +class McpClientTests(unittest.TestCase): + def setUp(self): + self.scratch = tempfile.TemporaryDirectory() + server = Path(self.scratch.name) / 'server.py' + server.write_text(FAKE_SERVER) + self.rows = [] + self.gateway = host.Gateway(sys.executable, [str(server)], dict(os.environ), Path(self.scratch.name) / 'gw.log', + record=self.rows.append) + self.gateway.initialize(timeout=30) + + def tearDown(self): + self.gateway.close(limit=10) + self.gateway.process.stdout.close() + self.gateway.process.stderr.close() + self.scratch.cleanup() + + def test_answers_errors_and_server_requests(self): + payload, latency = self.gateway.call('echo', {'workspace_id': 'off'}) + self.assertEqual(payload, {'workspace_id': 'off'}) + self.assertGreaterEqual(latency, 0) + refused = self.gateway.exec('off', ['busy']) + self.assertEqual((refused['outcome'], refused['code'], refused['attempt_id'], refused['process_id'], + refused['approval_id']), ('refused', 'WORKSPACE_BUSY', 'cs-attempt-0a', 'proc-2', 'appr-1')) + self.assertEqual(self.gateway.call('ping-me', {})[0], {'pinged': True}) + self.assertTrue(any(row['dir'] == 'in' for row in self.rows)) + + def test_no_answer_and_a_lost_connection(self): + with self.assertRaises(host.ToolError) as raised: + self.gateway.call('silent', {}, timeout=0.3) + self.assertEqual(raised.exception.code, host.NO_ANSWER) + result = self.gateway.exec('off', ['true'], timeout=0.3) + self.assertEqual((result['outcome'], result['code']), ('error', host.NO_ANSWER)) + with self.assertRaises(host.ToolError) as raised: + self.gateway.call('exit', {}, timeout=10) + self.assertEqual(raised.exception.code, analysis.TRANSPORT) + with self.assertRaises(host.ToolError) as raised: + self.gateway.call('echo', {}, timeout=10) + self.assertTrue(raised.exception.transport) + + +class HarnessTests(unittest.TestCase): + def setUp(self): + self.scratch = tempfile.TemporaryDirectory() + self.dir = Path(self.scratch.name) + + def tearDown(self): + for path in self.dir.rglob('*'): + os.chmod(path, 0o700 if path.is_dir() else 0o600) + self.scratch.cleanup() + + def test_consumer_settings_never_return_a_digest(self): + nested = textwrap.dedent(""" + schema = 1 + [consumers.dev-cli] + role = "workload" + credential_sha256 = "{0}" + [consumers.codespace] + generation = "codespace-g1" + role = "control_service" + credential_sha256 = "{0}" + max_instances = 1 + [consumers.codespace.control_reservation] + cpu_milli = 150 + memory_bytes = 536870912 + tasks = 40 + [projects] + """).format('a' * 64) + inline = textwrap.dedent(""" + [consumers.codespace] + generation = "codespace-g1" # comment + role = "control_service" + credential_sha256 = "{0}" + max_instances = 1 + control_reservation = {{ cpu_milli = 150, memory_bytes = 536870912, tasks = 40 }} + """).format('a' * 64) + expected = {'generation': 'codespace-g1', 'role': 'control_service', 'max_instances': 1, + 'control_reservation': RESERVATION} + for text in (nested, inline): + settings = qualify.consumer_settings(text) + self.assertEqual(settings, expected) + self.assertNotIn('a' * 64, json.dumps(settings)) + self.assertEqual(qualify.consumer_settings('[consumers.dev-cli]\nrole = "workload"\n'), {}) + + def test_documentation_keeps_the_candidate(self): + repo = self.dir / 'repo' + (repo / 'docs').mkdir(parents=True) + (repo / 'docs/guide.md').write_text('one\n') + (repo / 'main.rs').write_text('fn main() {}\n') + + def commit(): + subprocess.run(['git', 'add', '-A'], cwd=repo, env=GIT_ENV, check=True) + subprocess.run(['git', 'commit', '-q', '-m', 'c'], cwd=repo, env=GIT_ENV, check=True) + return qualify.code_tree(repo) + subprocess.run(['git', 'init', '-q'], cwd=repo, env=GIT_ENV, check=True) + first = commit() + (repo / 'docs/guide.md').write_text('two\n') + (repo / 'NOTES.md').write_text('notes\n') + self.assertEqual(commit(), first) + (repo / 'main.rs').write_text('fn main() { }\n') + self.assertNotEqual(commit(), first) + + def test_the_devguard_pin_from_the_lockfile(self): + (self.dir / 'Cargo.lock').write_text(textwrap.dedent(""" + [[package]] + name = "devguard-client" + version = "0.1.0" + source = "git+https://github.com/novelKR/DevGuard?rev=6e7e065#6e7e065" + + [[package]] + name = "serde" + version = "1.0.0" + """)) + self.assertEqual(qualify.devguard_lock(self.dir), {'devguard-client': { + 'version': '0.1.0', 'source': 'git+https://github.com/novelKR/DevGuard?rev=6e7e065#6e7e065'}}) + + def test_candidate_problems(self): + record = {'codespace': {'dirty': False, 'codex_patch': {'returncode': 0}}, + 'release': {'state': {'runs_release': True}, 'artifacts': {'devguardd': {'matches': True}}}, + 'policy': {'doctor': {'satisfied': True}, 'consumer': {'role': 'control_service'}}, + 'toolchain': {'rustc': f'rustc {qualify.PINNED_RUST} (abc 2026-01-01)'}} + self.assertEqual(qualify.candidate_problems(record), []) + record['policy']['consumer'] = None + record['release']['artifacts']['devguardd']['matches'] = False + self.assertEqual(qualify.candidate_problems(record), [ + 'an installed binary differs from the release manifest', + 'host.toml has no codespace control-service consumer']) + + def test_the_patched_codex_files_do_not_make_the_checkout_dirty(self): + def run(root, *args): + subprocess.run(['git', '-C', str(root), '-c', 'user.name=qualify', '-c', 'user.email=qualify@example.com', + '-c', 'commit.gpgsign=false', '-c', 'protocol.file.allow=always', *args], + check=True, capture_output=True) + + codex, checkout = self.dir / 'codex', self.dir / 'checkout' + for root in (codex, checkout): + root.mkdir() + run(root, 'init', '-q') + (codex / 'lib.rs').write_text('fn pinned() {}\n') + run(codex, 'add', 'lib.rs') + run(codex, 'commit', '-qm', 'pinned') + (checkout / 'Cargo.toml').write_text('[workspace]\n') + run(checkout, 'submodule', 'add', '-q', str(codex), 'third_party/codex') + run(checkout, 'add', 'Cargo.toml') + run(checkout, 'commit', '-qm', 'gitlink') + self.assertFalse(qualify.dirty(checkout)) + # The recorded patch changes the submodule's files; the Codex check judges them. + (checkout / 'third_party/codex/lib.rs').write_text('fn pinned() { observed() }\n') + self.assertFalse(qualify.dirty(checkout)) + (checkout / 'Cargo.toml').write_text('[workspace]\nmembers = []\n') + self.assertTrue(qualify.dirty(checkout)) + run(checkout, 'checkout', '-q', 'Cargo.toml') + run(checkout / 'third_party/codex', 'commit', '-qam', 'another commit') + self.assertTrue(qualify.dirty(checkout)) + + def test_a_late_action_misses_slots_and_never_runs_late(self): + writer = host.Writer() + + def action(slot): + if slot == 1: + time.sleep(0.45) + return {'outcome': 'answered', 'latency_ms': 1.0, 'state': 'running'} + schedule = qualify.Schedule(writer, self.dir / 's.jsonl', action, time.monotonic() + 0.05, period=0.1) + schedule.start() + time.sleep(0.9) + rows = schedule.stop() + writer.close() + missed = {row['slot'] for row in rows if row['outcome'] == 'missed_slot'} + self.assertTrue({2, 3, 4} <= missed, rows) + self.assertEqual([row['slot'] for row in rows], list(range(len(rows)))) + self.assertEqual(len(qualify.read_rows(self.dir, 's')), len(rows)) + + def test_slots_that_pass_while_the_last_action_runs_are_missed(self): + writer = host.Writer() + + def action(slot): + if slot == 2: + time.sleep(0.5) + return {'outcome': 'answered', 'latency_ms': 1.0, 'state': 'running'} + began = time.monotonic() + schedule = qualify.Schedule(writer, self.dir / 's.jsonl', action, began, period=0.1) + schedule.start() + time.sleep(0.36) + rows = schedule.stop() + writer.close() + self.assertEqual([row['outcome'] for row in rows[:3]], ['answered'] * 3) + self.assertEqual([row['outcome'] for row in rows[3:]], ['missed_slot']) + + def test_offsets_end_by_themselves(self): + writer = host.Writer() + schedule = qualify.Schedule(writer, self.dir / 'o.jsonl', lambda slot: {'outcome': 'refused'}, + time.monotonic(), offsets=(0.0, 0.05)) + schedule.start() + schedule.join(timeout=5) + self.assertFalse(schedule.is_alive()) + self.assertEqual(len(schedule.stop()), 2) + writer.close() + + def test_phase_expectations(self): + self.assertTrue(qualify.phase_checks('admitted', admitted(4))[0]['passed']) + self.assertFalse(qualify.phase_checks('admitted', admitted(3) + refused(1))[0]['passed']) + self.assertTrue(qualify.phase_checks('refused', refused(4, 'ADMISSION_UNKNOWN'))[0]['passed']) + self.assertFalse(qualify.phase_checks('refused', refused(4))[0]['passed']) + self.assertFalse(qualify.phase_checks('refused', [{'slot': 0, 'outcome': 'missed_slot'}])[0]['passed']) + + def test_repetition_outcomes(self): + report = {'service_before': {'runs_release': True, 'service_pid': 7}, + 'service_after': {'runs_release': True, 'service_pid': 7}, 'rehearsal': False} + good = {'verdict': 'pass', 'reasons': []} + self.assertIsNone(qualify.early_outcome(report)) + self.assertEqual(qualify.repetition_outcome(report, good, good), ('pass', [])) + restarted = dict(report, service_after={'runs_release': True, 'service_pid': 8}) + self.assertEqual(qualify.repetition_outcome(restarted, good, good)[0], 'inconclusive') + self.assertEqual(qualify.repetition_outcome(dict(report, rehearsal=True), good, good)[0], 'inconclusive') + cold = dict(report, prewarm=[{'result': 'completed', 'exit_code': 1}]) + self.assertEqual(qualify.early_outcome(cold)[1], ['the warm target directory could not be built']) + + def test_modes(self): + self.assertEqual(qualify.modes('uds'), ['uds']) + self.assertEqual(qualify.modes('in-process, uds'), ['in-process', 'uds']) + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + qualify.arguments(['matrix', '--out', 'x', '--mcp-bin', 'm', '--runtime-bin', 'r', '--release', 'v', + '--modes', 'pipe']) + + # ------------------------------------------------------------ verify + + def write_rows(self, directory, rows): + directory.mkdir(parents=True, exist_ok=True) + for name, values in rows.items(): + (directory / f'{name}.jsonl').write_text(''.join(json.dumps(row) + '\n' for row in values)) + + def scenarios_run(self, name='matrix', falsify=False): + evidence = host.Evidence(self.dir / name) + check = analysis.check + entries = [{'scenario': 'exits', 'mode': 'uds', 'io': 'pipe', 'checks': [check('a', True)]}, + {'scenario': 'timeout', 'mode': 'uds', 'io': 'pty', 'checks': [check('a', True), check('b', False)]}] + for entry in entries: + entry['verdict'] = analysis.scenario_verdict(entry['checks']) + evidence.append('scenarios.jsonl', entry) + if falsify: + entries[1]['verdict'] = 'pass' + verdict = analysis.suite_verdict([entry['verdict'] for entry in entries]) + evidence.write_json('report.json', {'suite': 'matrix', 'verdict': verdict, 'result': {'verdict': verdict}, + 'harness': qualify.harness()}) + if falsify: + os.chmod(evidence.path / 'scenarios.jsonl', 0o600) + (evidence.path / 'scenarios.jsonl').write_text(''.join(json.dumps(entry) + '\n' for entry in entries)) + evidence.seal() + return evidence.path + + def test_verify_scenarios(self): + result = qualify.verify(self.scenarios_run()) + self.assertEqual((result['verdict'], result['recomputed_verdict']), ('verified', 'failed'), result) + self.assertTrue(result['same_harness']) + falsified = qualify.verify(self.scenarios_run('falsified', falsify=True)) + self.assertEqual(falsified['verdict'], 'failed') + self.assertTrue(any(problem.startswith('uds pty timeout verdict') for problem in falsified['problems']), + falsified['problems']) + + def authority_run(self, name='authority', lie=False): + evidence = host.Evidence(self.dir / name) + records = [] + for phase, fault, expectation, release_expected in qualify.AUTHORITY_PHASES: + raw = evidence.file(f'uds/authority/{phase}/x').parent + terminations, releases = terminated(3) + if not release_expected: + releases = [] + rows = {'status_pipe': answered(15), 'status_pty': answered(15), 'terminations': terminations, + 'releases': releases, 'io': echoed(5), + 'refusals': admitted(4) if expectation == 'admitted' else refused(4, 'ADMISSION_UNKNOWN')} + self.write_rows(raw, rows) + record = {'mode': 'uds', 'phase': phase, 'fault': fault, 'seconds': 15.01, + 'release_expected': release_expected, 'expectation': expectation, 'invalid': [], + 'checks': qualify.phase_checks(expectation, rows['refusals']), 'raw': qualify.raw_hashes(raw)} + record['verdict'], record['reasons'], record['metrics'] = qualify.phase_verdict(record, rows) + if lie and phase == 'stopped': + record['metrics']['status']['p99_ms'] = 1.0 + evidence.write_json(f'uds/authority/{phase}/phase.json', record) + records.append(record) + verdict = analysis.suite_verdict([record['verdict'] for record in records]) + evidence.write_json('report.json', {'suite': 'authority', 'verdict': verdict, 'result': {'verdict': verdict}, + 'harness': qualify.harness()}) + evidence.seal() + return evidence.path + + def test_verify_authority(self): + path = self.authority_run() + result = qualify.verify(path) + self.assertEqual((result['verdict'], result['recomputed_verdict']), ('verified', 'passed'), result) + self.assertEqual(qualify.verify(self.authority_run('lie', lie=True))['verdict'], 'failed') + rows = path / 'uds/authority/paused/status_pty.jsonl' + os.chmod(rows, 0o600) + rows.write_text(rows.read_text().replace('"latency_ms": 5.0', '"latency_ms": 900.0', 1)) + result = qualify.verify(path) + self.assertEqual(result['verdict'], 'failed') + self.assertTrue(any(problem.startswith('changed uds/authority/paused/status_pty.jsonl') + for problem in result['problems']), result['problems']) + + def control_run(self, name='control', interval_seconds=60.2): + evidence = host.Evidence(self.dir / name) + plan = dict(qualify.CONTROL_REHEARSAL, combinations=['in-process'], load='cold', rehearsal=True) + evidence.write_json('run.json', {'plan': plan, 'protocol': qualify.CONTROL_PROTOCOL, + 'targets': {'status_p99_ms': analysis.STATUS_P99_MS, + 'termination_ack_p99_ms': analysis.TERMINATION_P99_MS}, + 'release': {'id': 'r1'}, 'reservation': RESERVATION}) + key = 'in-process-1' + records = {} + for kind in ('idle', 'load'): + raw = evidence.file(f'{key}/{kind}/x').parent + terminations, releases = terminated(3) + rows = {'status_pipe': answered(60), 'status_pty': answered(60), 'terminations': terminations, + 'releases': releases, 'io': echoed(12), 'refusals': refused(2)} + sampled = {'host': [{'pressure_level': '1', 'swap_used_mib': 10.0, 'load_average': [2.0, 1.0, 1.0]}], + 'service': [{'runs_release': True, 'current': 'r1', 'service_pid': 7}], + 'footprint': [{'owner': {'rss_bytes': 1 << 20, 'cpu_percent': 1.0, 'threads': 9}, + 'gateway': None}]} + self.write_rows(raw, dict(rows, **sampled)) + runs = [] + record = {'kind': kind, 'combination': 'in-process', 'invalid': [], 'seconds': interval_seconds, + 'started_unix': 1000.0, 'ended_unix': 1000.0 + interval_seconds, + 'status_targets': [{'effective': True}, {'effective': True}], + 'journal': {'still_charged': [], 'duplicate_launches': 0, 'readmitted_uncertain': 0}} + if kind == 'load': + runs = [{'consumer': 'cargo', 'result': 'completed', 'exit_code': 0, 'running_from': 1000.0, + 'ended': 1000.0 + interval_seconds}] + (raw / 'runs.json').write_text(json.dumps(runs) + '\n') + record.update(consumers=['cargo'], load_issues=[], load_failures=[]) + record['raw'] = qualify.raw_hashes(raw) + metrics = qualify.interval_metrics(record, rows, sampled, runs, 'r1', 7, RESERVATION) + record['verdict'], record['reasons'] = analysis.interval_verdict(kind, record['invalid'], metrics) + record['metrics'] = metrics + evidence.write_json(f'{key}/{kind}/interval.json', record) + records[kind] = record + report = {'combination': 'in-process', 'repetition': 1, 'rehearsal': True, + 'service_before': {'runs_release': True, 'service_pid': 7}, + 'service_after': {'runs_release': True, 'service_pid': 7}, + 'intervals': {kind: {'verdict': record['verdict'], 'reasons': record['reasons']} + for kind, record in records.items()}} + report['verdict'], report['reasons'] = qualify.repetition_outcome(report, records['idle'], records['load']) + evidence.write_json(f'{key}/report.json', report) + combination = analysis.combination_verdict([report['verdict']], 1) + overall = analysis.overall_verdict({'in-process': {'verdict': combination}}, plan, qualify.CONTROL_PROTOCOL, + True) + evidence.write_json('report.json', { + 'suite': 'control', 'verdict': overall, 'harness': qualify.harness(), + 'result': {'verdict': overall, 'combinations': {'in-process': { + 'verdict': combination, 'repetitions': [{ + 'repetition': 1, 'verdict': report['verdict'], 'reasons': report['reasons'], + 'report_sha256': host.sha256(evidence.path / key / 'report.json')}]}}}}) + evidence.seal() + return evidence.path, records + + def test_verify_control(self): + path, records = self.control_run() + self.assertEqual([records[kind]['verdict'] for kind in ('idle', 'load')], ['pass', 'pass']) + result = qualify.verify(path) + self.assertEqual((result['verdict'], result['recomputed_verdict']), ('verified', 'inconclusive'), result) + with contextlib.redirect_stdout(io.StringIO()) as printed: + self.assertEqual(qualify.main(['verify', str(path)]), 0) + self.assertEqual(json.loads(printed.getvalue())['verdict'], 'verified') + interval = path / 'in-process-1/load/interval.json' + os.chmod(interval, 0o600) + record = json.loads(interval.read_text()) + record['metrics']['status']['p99_ms'] = 1.0 + interval.write_text(json.dumps(record)) + os.chmod(path / 'MANIFEST.json', 0o600) + manifest = json.loads((path / 'MANIFEST.json').read_text()) + manifest['files']['in-process-1/load/interval.json'] = host.sha256(interval) + (path / 'MANIFEST.json').write_text(json.dumps(manifest)) + result = qualify.verify(path) + self.assertEqual(result['verdict'], 'failed') + self.assertTrue(any(problem.startswith('in-process-1 load metrics') for problem in result['problems']), + result['problems']) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(qualify.main(['verify', str(path)]), 1) + + def test_verify_refuses_an_unsealed_directory(self): + (self.dir / 'open').mkdir() + self.assertEqual(qualify.verify(self.dir / 'open')['verdict'], 'failed') + + +if __name__ == '__main__': + unittest.main() From ca3670bc006ec55bb6c6ec1dc5f5998248af3b20 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 17:06:42 +0000 Subject: [PATCH 2/4] docs: CS-RG qualification protocol and the legacy off path decision (CSRG-U6) docs/cs-rg-qualification.md, in English and Korean, is the protocol the harness implements: the candidate, the supported-mode claims, the functional matrix and fault cases with their pass rules, how the order's fault list is covered and where it is not, the control-path targets, schedules and verdicts, accounting, regressions and evidence. It also records the decision the work order requires for the existing backends (CSRG-C09): keep participation off as the compatibility path and default, with its scope, duplicated parts, what it does not provide, and when to revisit or remove it. The integration roadmap links the page from its status and from the C09 paragraph. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/cs-rg-qualification.md | 226 ++++++++++++++++++++++++++++++++ docs/devguard-integration.md | 4 +- docs/ko/cs-rg-qualification.md | 226 ++++++++++++++++++++++++++++++++ docs/ko/devguard-integration.md | 4 +- docs/translations.json | 59 ++++++++- 5 files changed, 513 insertions(+), 6 deletions(-) create mode 100644 docs/cs-rg-qualification.md create mode 100644 docs/ko/cs-rg-qualification.md diff --git a/docs/cs-rg-qualification.md b/docs/cs-rg-qualification.md new file mode 100644 index 0000000..bad91aa --- /dev/null +++ b/docs/cs-rg-qualification.md @@ -0,0 +1,226 @@ + + +# CS-RG qualification protocol + +[English](cs-rg-qualification.md) | [ํ•œ๊ตญ์–ด](ko/cs-rg-qualification.md) + +This page is the protocol for qualifying CodeSpace's use of DevGuard (CS-RG): which combination is qualified, which modes are claimed, how each claim is tested and measured, what makes a run pass, and what evidence a run leaves. It is frozen together with its harness, [`scripts/qualify-devguard.py`](../scripts/qualify-devguard.py), before any qualification run. A change to it, or to the harness, after the first qualification run is a new protocol: it is reviewed like any other change, and the runs it affects are repeated. The [integration roadmap](devguard-integration.md) describes what each unit built; this page decides whether the result qualifies. + +**Status:** the protocol and the harness exist; no qualification run has been made, and nothing here is a qualification result. CS-RG stays in progress and not qualified until a run under this protocol passes and its evidence is reviewed. + + + +## The candidate + +A run qualifies one exact combination, recorded by the harness's `candidate` suite in `candidate.json`, never by hand: + +| Part | Recorded as | +| --- | --- | +| CodeSpace | the commit, whether tracked files outside the Codex submodule differ from it, and the SHA-256 of its tree without `docs/`, `docs-site/` and `*.md` files (the Codex gitlink included) | +| DevGuard client | the `devguard-*` entries of `Cargo.lock` ([client pin](upstream-lock.md#devguard-client-pin)) | +| Codex | the gitlink and the result of `scripts/codex_patch.py check` ([recorded patch](upstream-lock.md#codex-downstream-patch)) | +| Binaries | SHA-256 and size of `codespace-mcp` (built with `-p codespace-server --features devguard`) and the worker `codespace-codex-runtime` (`--features devguard`), and of the fixture, helper and `devguard-qualify` binaries when given | +| DevGuard release | the installed release ID, the SHA-256 of its `MANIFEST.json`, whether each installed binary matches it, and the service's status | +| Policy | the SHA-256 of `host.toml`, `devguard doctor --require admission,registration,macos-cooperative` (configuration fingerprint, capabilities, protocol), and the `codespace` consumer's role, generation, instance limit and control reservation. The consumer's credential digest is never read into the record | +| DG-1 qualification | DevGuard's qualification record of the release, when it exists: its SHA-256, its verdicts, and whether it was measured under the same `host.toml`, fingerprint and capabilities | +| Toolchain and host | `rustc -vV`, `cargo -V`, Python, Xcode; macOS version, model, logical CPUs, memory, boot time; the service's work capacity and pressure state | + +`candidate` passes when the checkout has no uncommitted changes outside the Codex submodule, the Codex check passes (the submodule holds the pinned commit with exactly the recorded patch), the service runs the release, every installed binary matches the manifest, `doctor` is satisfied, `host.toml` has a `codespace` consumer with the `control_service` role, and Rust 1.95.0 is first in `PATH`. A final report also needs a DG-1 qualification of the same release under the same policy; `candidate` records it without requiring it, so that the harness can run before it exists. + +Any change to code, binaries, the release, the policy, Codex or the toolchain is a new candidate and repeats the runs it affects. A commit that changes only documentation keeps the candidate, and the code-tree hash shows it. + + + +## Supported modes + +Each claimed cell is tested on its own; no cell is inferred from another. + +| Platform | Runner | I/O | Participation | Claim | Shown by | +| --- | --- | --- | --- | --- | --- | +| macOS, installed service with native host evidence | `in-process` | pipe, PTY | `required` | qualify | the functional matrix, the fault cases and the control-path measurement | +| macOS | `uds` with `--runtime-bin` (the worker owns executions) | pipe, PTY | `required` | qualify | the same, with the worker registered | +| macOS | `uds` with `--runner-socket` | pipe, PTY | `required` | unsupported | refused `RESOURCE_POLICY_UNSUPPORTED` before an attempt exists | +| macOS | any | pipe, PTY | `required` under `--devguard status` | unsupported | refused `RESOURCE_POLICY_UNSUPPORTED` before an attempt exists | +| macOS | supported | pipe, PTY | `required` without a usable launch helper | unsupported | admitted, then refused `MANAGED_LAUNCH_UNAVAILABLE` with the attempt cancelled | +| macOS | supported | pipe, PTY | `off` and `required` workspaces in one gateway | qualify | concurrent governed and ungoverned children, each holding only its standard descriptors | +| macOS, Linux | all | pipe, PTY | `off` | existing behavior | the [regression stages](#regressions), with and without the `devguard` feature | +| Linux | all | pipe, PTY | `required` | unsupported until DG-LINUX | refused before anything runs: `RESOURCE_AUTHORITY_UNAVAILABLE` under `--devguard register`, since the owner cannot register without native host evidence, otherwise `RESOURCE_POLICY_UNSUPPORTED`; shown by Linux CI | + +A gateway built without the `devguard` feature has no DevGuard flags; `required` workspaces there are refused as with `--devguard status`. + + + +## Running the harness + +The harness runs on the qualification host, a Mac with the release installed and the `codespace` consumer provisioned, with the system Python (3.9 or later) and no third-party packages. Every process it starts gets a minimal environment (`HOME`, `PATH=/usr/bin:/bin:/usr/sbin:/sbin`, `TMPDIR`, `LANG=C` and the `CODESPACE_*` settings it names), and the harness itself is started the same way: + +```bash +env -i HOME="$HOME" TMPDIR="$TMPDIR" PATH="/bin:/usr/bin:/bin:/usr/sbin:/sbin" \ + /usr/bin/python3 scripts/qualify-devguard.py matrix --out evidence/ \ + --mcp-bin --runtime-bin --release +``` + +| Suite | Against | Extra arguments | +| --- | --- | --- | +| `candidate` | the installed service | `--release`; optionally `--fixture-bin`, `--fixture-helper`, `--qualify-bin` to record them | +| `matrix` | the installed service | `--release` | +| `faults` | DevGuard's fixture authority | `--fixture-bin` (`crates/devguard` example `fixture_authority`), `--fixture-helper` (`devguard-launch` built from the client pin) | +| `authority` | DevGuard's fixture authority | as `faults` | +| `control` | the installed service under DG-1's load | `--release`, `--work` (a new directory), `--qualify-bin` (DevGuard's `devguard-qualify`), `--devguard-checkout` (a DevGuard checkout holding the release's source commit), `--load cold` or `warm`, `--rehearsal`, `--repetitions` | +| `verify` | a sealed evidence directory | the directory | + +`--modes` limits a suite to `in-process` or `uds`; both run by default. The installed-service suites default to the service's socket, journal, log, `host.toml` and the consumer's credential file, whose path is handed to the gateway and never read by the harness. Exit status: 0 passed or qualified, 1 failed, 2 incomplete or inconclusive, 3 the harness failed, 130 interrupted; `verify` exits 0 when the run checks out and 1 when it does not. + +The harness talks to `codespace-mcp` over standard input and output as any MCP client does (protocol `2025-11-25`), with its own small client, so a timing is the tool call's local round trip from the harness and includes no network. It reads DevGuard's journal read-only, as DevGuard's own measurement does, and takes a governed launch's attempt ID and scope root from the gateway's log. + + + +## Functional matrix + +`matrix` runs against the installed service, once per runner mode, after checking that the service runs the release and nothing is charged. Governed workspaces request 50 mCPU, 64 MiB and 8 tasks; the gateway's process timeout is 15 seconds. It never runs a command that leaves a descendant, escapes its group or restarts the service, so that no attempt can stay Suspect on the owner's service. + +| Scenario | Pipe and PTY | Passes when | +| --- | --- | --- | +| registration | | `workspace_info` reports participation `registration`, the authority available, the owner (`in_process` or `worker`) registered, governing executions, and DevGuard's one active `codespace` instance is that process | +| exits | each | exit 0 and exit 3 are reported as exited with their codes; a SIGTERM death is `signaled` with signal 15 on a pipe, and exited with code 1 on a PTY, as the pinned Codex PTY reports it ([process status](agent-integration.md)) | +| io | each | on a pipe, `write_stdin` reaches `cat` and its output returns, `process_resize` is refused `PROCESS_NOT_TTY` and termination is reported; on a PTY, the terminal starts at 24ร—80, a resize to 40ร—100 is answered and seen by the process, and stdin reaches it | +| termination | each | `terminate_process` answers, the process is reported terminated and its scope's process group is gone | +| timeout | each | a `sleep 600` is reported `timeout` | +| exec-failed | each | a script whose interpreter is missing or not executable is refused `PROCESS_SPAWN_FAILED` with errno 2 or 13; the helper claimed the grant and DevGuard released it when the scope ended | +| shortage | each | three calls to a workspace asking for more CPU than the work capacity are each refused `RESOURCE_UNAVAILABLE` as new, denied attempts holding nothing, never `WORKSPACE_BUSY` | +| unsupported-minimum | each | a workspace whose minimum asks the kernel to enforce tasks is refused `RESOURCE_POLICY_UNSUPPORTED`, denied by DevGuard | +| mixed | each | an ungoverned and a governed execution run at once in one gateway, and each child holds only descriptors 0, 1 and 2 | +| no-helper, unusable-helper | | a gateway without a launch helper, or with one that does not exist, refuses `MANAGED_LAUNCH_UNAVAILABLE`, DevGuard records the attempt cancelled, and `off` workspaces still run | +| status-mode | | under `--devguard status` a governed execution is refused `RESOURCE_POLICY_UNSUPPORTED` before an attempt exists; `off` still runs | +| authority-unavailable | | with no authority at the socket, it is refused `RESOURCE_AUTHORITY_UNAVAILABLE`, DevGuard holds no attempt for it, and `off` still runs | +| runner-socket | `uds` only | with a worker the gateway did not start, governed executions are refused `RESOURCE_POLICY_UNSUPPORTED` before an attempt exists | +| settled | | nothing of the consumer stays charged and no instance stays active; otherwise the matrix stops | + +Every governed launch is then checked against DevGuard's own record of its attempt ([accounting](#accounting)). + + + +## Fault cases + +`faults` injects the faults that would leave the installed service with Suspect or charged work, so it runs them against DevGuard's fixture authority: DevGuard's daemon code at CodeSpace's client pin, in a private state directory, provisioning its own `codespace` consumer. Each case gets a fresh fixture authority; governed workspaces request 50 mCPU, 32 MiB and 4 tasks. + +| Case | Pipe and PTY | Passes when | +| --- | --- | --- | +| descendant | each | a root that exits leaving a descendant keeps its attempt charged, not Suspect, while the descendant runs, and the attempt is released once it ends. On a PTY the reconciler is paused around the exit, so only the observation before the reap can adopt the descendant | +| escape | each | a descendant that leaves the scope's group with `setsid` keeps the attempt charged as Suspect after everything has ended | +| stalled-observer | each | with the reconciler paused and the authority held past the observation's bound, CodeSpace reaps the root before the hold ends, and the unproven descendant leaves the attempt Suspect, also after it ends | +| authority-lost | | with the authority killed while a pipe and a PTY execution run, both still answer `running` and are terminated within 4 seconds, a new governed execution is refused `RESOURCE_AUTHORITY_UNAVAILABLE` with nothing at DevGuard, and `off` still runs | +| authority-restart | | with the authority restarted while an approved, resumed execution runs, the process stays CodeSpace's and runs to its end, its attempt stays charged as Suspect, resuming the approval again returns the same process with one launch, and a new execution is admitted and released | +| authority-stalled | | with the authority's lock held past the owner's deadlines, the call is refused `ADMISSION_UNKNOWN` or `RESOURCE_AUTHORITY_UNAVAILABLE` before the hold ends, the command never runs, DevGuard never hands the attempt to a helper and nothing of it stays charged, and the next execution is a new attempt that runs | +| helper-ends | | a helper that exits before READY makes the call refused `PROCESS_SPAWN_FAILED` (ended before it reported ready), the command never runs, and DevGuard releases the grant as never started (`no_helper_created`) | +| credential | | with a credential the authority refuses and with a missing one, governed executions are refused `RESOURCE_AUTHORITY_UNAVAILABLE` naming `credential_refused` or `credential_unavailable`, and `off` still runs | +| terminate-race | | twelve terminations around the process's own exit, alternating pipe and PTY, each end once as exited or terminated, are launched once and released | +| shutdown | | when the gateway's input closes with a governed pipe and PTY execution running (in `uds` mode the worker goes with it), the gateway and the owner exit, both scopes end and are released, and a new gateway registers and runs without launching either again | +| worker-killed | `uds` only | with the worker killed while it prepares a governed execution, the gateway answers the call, the command never runs, DevGuard never hands the attempt to a helper and holds nothing of it, and the gateway keeps running | + + + +### Coverage of the order's fault list + +| Fault | Where it is shown | +| --- | --- | +| Admission refusal | matrix `shortage` and `unsupported-minimum` (installed service, both modes, pipe and PTY) | +| Authority gone before admission | matrix `authority-unavailable`; faults `credential` | +| Authority gone after start | faults `authority-lost`, `authority-restart`; `authority` suite | +| Lost launch answer | faults `authority-stalled` for the request a session is waiting on. A lost answer to a request already past registration cannot be produced from outside the owner; DevGuard's client tests and CodeSpace's runner and gateway tests with a scripted worker cover it | +| Helper failure | matrix `no-helper`, `unusable-helper`; faults `helper-ends` | +| READY, then the program cannot run | matrix `exec-failed` (errno 2 and 13, pipe and PTY) | +| Observer timeout | faults `stalled-observer` | +| Surviving descendant | faults `descendant`, `escape` | +| Terminate and exit race | faults `terminate-race` | +| Shutdown and exit race | faults `shutdown` | +| Runner connection lost | faults `worker-killed` while preparing; a worker lost during the launch itself is covered by the gateway tests with a scripted worker | +| Stale or duplicate request | faults `authority-restart` (resume again returns the same process, one launch) and the control measurement's duplicate-launch count. A replay of a preparation or launch message on the runner wire is not tested; this is a known gap | +| Uncertain execution | faults `authority-stalled`; the control measurement fails when an uncertain attempt is admitted again | +| Credential failure | faults `credential` | +| Journal or storage refusal | not injected: the fixture authority cannot make its journal refuse. DevGuard's own suites cover the refusal, and CodeSpace's tests cover how its codes map | + +The gateway tests that use the fixture authority run only on macOS with the DevGuard test binaries, in the [regression stages](#regressions) on the qualification host. macOS CI runs the adapter's tests, the runner's DevGuard tests and the gateway's `devguard` module tests, but not those gateway tests. + + + +## Control-path availability + +The targets are DevGuard's established ones that apply to the control path ([SLO protocol](https://github.com/novelKR/DevGuard/blob/ea98d7fadf6ac10e9393db923105bf31f719952a/docs/planning/verification.md#slo-protocol)), not redefined here: local `process_status` p99 at most 500 ms; termination acknowledgement p99 at most 1 second, with the scope's actual end reported separately; no MCP connection lost; no execution launched twice; no uncertain execution started again. The foreground browser targets are DG-1's and are not measured here. Rejection latency of a new governed execution and output responsiveness have no established target; they are measured and reported, with the design's 250 ms preparation budget as the reference. + + + +### Intervals and samples + +`control` measures each runner mode (`in-process` and `uds`) in three repetitions. A repetition is a 10-minute idle interval followed by a load interval of at least 30 minutes. Each interval starts a new gateway registered with the installed service, whose governed workspaces request 25 mCPU, 16 MiB and 2 tasks each, and samples on fixed schedules: + +| Schedule | Every | What is timed | +| --- | --- | --- | +| status | 1 s, two targets | `process_status` of a running governed `cat` (pipe) and `sleep` (PTY) | +| termination | 20 s | a fresh governed `sleep`, alternating pipe and PTY: `terminate_process` to its answer; separately, to the exited status, to the scope's process group gone, and to DevGuard's journal showing the attempt released | +| I/O | 5 s | a numbered line written with `write_stdin` to the `cat` until `read_process` returns it | +| refusal | 30 s | a governed execution in a workspace asking for more CPU than the work capacity, which must be refused | +| host | 5 s | memory pressure level, swap and load average | +| service | 60 s | whether the service still runs the release, and its process ID | +| footprint | 5 s | the execution owner's RSS, threads, processes and CPU (and the gateway's in `uds` mode), compared with the consumer's control reservation and reported | + +A slot whose moment passes while the previous sample still runs is recorded as missed, never taken late. A missed slot, an error, or an answer about a target that no longer runs is a missing sample, and a missing sample ranks above every value in the nearest-rank percentile, as in DG-1. A termination target refused for capacity or pressure (`RESOURCE_UNAVAILABLE`) is not a sample. At the end of the interval both status targets are terminated, and each termination must be effective. + + + +### Load + +The load is DG-1's: the six consumers of DevGuard's `scripts/measure.py`, from the given DevGuard checkout, run through the installed release's `devguard exec` with `devguard-qualify` and a Cargo build of the release's source, so that pressure and queueing are the service's own. `--load cold` (the default) builds from empty target directories; `--load warm` builds a target directory first and is inconclusive if that build fails. A qualification run uses the cold condition; a warm run is an additional condition, reported separately when it is made. The consumers run until the interval's planned end and then finish their current runs within DG-1's completion limit. The harness's targets and the load fit in half the work capacity, the capacity DevGuard keeps under memory warning. + + + +### Verdicts + +An interval is **inconclusive** (an invalid observation, never a pass) when other work was charged as it began, the journal could not be read, a status target could not be started, raw evidence could not be written, the service could not be observed or had another release selected, or the load did not finish within its limit. + +Otherwise it **fails** when the status p99 exceeds 500 ms; the acknowledgement p99 exceeds 1 second; a termination was acknowledged without ending its target; a terminated target was not released; an I/O line never came back; any sample lost the MCP connection; a status target's final termination was not effective; the measured release restarted or became unhealthy; a load run is uncertain or a load attempt stays charged; or a harness attempt stays charged, an execution was launched more than once, or an uncertain attempt was admitted again. + +It is also **inconclusive** when too little was observed: fewer status samples than 90% of the schedule, fewer termination samples than 80%, or a load that was not DG-1's (a consumer with no successful run, one that ran for less than a quarter of the interval, one with failed runs, or admitted load running for less than half of it). Otherwise it **passes**. + +A repetition passes when both intervals pass and the service still runs the release with the same process afterwards; a failing idle interval makes it inconclusive, never a failure of the load. A runner mode qualifies when all three repetitions pass, and fails when any fails; values are never pooled across repetitions. The run qualifies only under the full protocol (both modes, three repetitions, 10 and 30 minutes) with both modes qualified, and fails when either fails. `--rehearsal` (30 seconds idle, 60 seconds load, one repetition) tries the run and never qualifies. + + + +### Authority failure + +`authority` measures process control while the fixture authority is unhealthy, in five 15-second phases per runner mode, against the same running status targets: `baseline`; `paused` (the reconciler paused); `stalled` (its lock held for the whole phase); `restarted`; `stopped` (its process killed). Status is sampled every second, the I/O echo every 3 seconds, three governed targets started before the fault are terminated at 3, 7 and 11 seconds, and a new governed execution is attempted at 1, 5, 9 and 13 seconds. Each phase is judged by the interval rules above, and new executions must be admitted while the authority can answer (`baseline`, `paused`, `restarted`) and refused `RESOURCE_AUTHORITY_UNAVAILABLE` or `ADMISSION_UNKNOWN` while it cannot (`stalled`, `stopped`). After a restart or once stopped, DevGuard cannot release targets started before the fault: an unreleased target is reported there, not failed. + + + +## Resource accounting + +The harness checks DevGuard's record, never CodeSpace's own view. For each governed launch in the matrix it reads the attempt the gateway logged from the journal: the reserved and applied quantities equal the workspace's request; CPU, memory and tasks are each `applied`; the scope root is the process the gateway launched; and once the scope ends the attempt is released with `scope_terminated`, holds nothing and its tracking stayed complete. A refused call names its attempt, which DevGuard records in the expected phase (`denied`, `cancelled` or `released`) holding nothing, or no attempt exists where the refusal comes first. Under the control load, the measured footprint of the gateway and the worker is compared with the `codespace` reservation and reported; the reservation is DevGuard's accounting estimate, so the comparison is not a verdict. + + + +## Regression and parity + +On the candidate, on the qualification host in the minimal environment, `python3 scripts/validate-upstream.py all macos-core macos-fork-race` runs every stage, with the DevGuard test binaries required in the stages that use them; Linux is shown by CI on the candidate head, whose `dependencies` stage checks the dependency graph with the `devguard` feature on and off. A regression in any stage fails the qualification. + + + +## Evidence + +Every suite but `verify` writes a new directory and seals it: `report.json` (the suite, its arguments, the harness's file hashes and commit, the verdict, and what was settled at the end), the suite's records, and the raw samples as JSON lines, including each gateway's MCP transcript and log. Sealing records every file's SHA-256 in `MANIFEST.json` and makes the files read-only; failed, inconclusive and interrupted runs are sealed and kept too. Nothing in a directory holds a credential, a credential digest or a child's environment. + +`qualify-devguard.py verify ` checks the manifest and recomputes every verdict from the raw files: scenario verdicts from their checks; each authority phase and each control interval from its samples; then each repetition, runner mode and the run. It reports whether the harness that ran is the one verifying, and writes nothing. + + + +## Legacy `off` path decision + +This is the decision record that the work order requires for the existing backends (CSRG-C09). + +**Decision:** keep participation `off` as the compatibility path and default, unchanged. + +- **Scope.** Participation `off`, the default and what a missing setting means, on every platform and runner mode, pipe and PTY. Its executions are the request's own command: pipe children have CodeSpace's single reaper, which governed executions share; PTY children use the pinned Codex PTY spawn without the observation window before the reap. +- **Why.** The work order does not migrate code for neatness alone. `off` is the product default; `required` is an operator's explicit choice, unsupported on Linux until DG-LINUX, so `off` is the only path there. Making governed launching the only path would remove every Linux execution and every setup without DevGuard. +- **What is duplicated.** One spawn path serves both and chooses the program: the request's command, or DevGuard's launch helper with its permit and report descriptors (`How::Direct` and `How::Managed` in [`crates/runner/src/process.rs`](../crates/runner/src/process.rs)). Reaping is not duplicated: a pipe child of either kind has CodeSpace's one reaper, and a PTY child the pinned Codex PTY's waiter. A PTY differs in one call, `spawn_inheriting` for ungoverned executions and `spawn_observed`, from the [recorded patch](upstream-patches.md), for governed ones. The governed path adds admission, launch and settlement ([`crates/runner/src/launch.rs`](../crates/runner/src/launch.rs)), which `off` never enters. Without the `devguard` feature no DevGuard package is in the build; with it, only DevGuard's client and contract crates are, as the `dependencies` stage checks. +- **What `off` does not provide.** No admission, reservation or accounting at DevGuard, no scope policing, and no observation of an ungoverned PTY root before its reap. Only `off` executions can use the Linux command sandbox; governed ones are refused where it is on. +- **Revisit.** When DG-LINUX qualifies Linux enforcement, when a Codex pin move changes PTY spawning, or when `required` becomes the default. +- **Removal.** Only with an independent product reason and regression evidence that every supported mode keeps its behavior without it. diff --git a/docs/devguard-integration.md b/docs/devguard-integration.md index 037542e..7b54b68 100644 --- a/docs/devguard-integration.md +++ b/docs/devguard-integration.md @@ -8,7 +8,7 @@ [DevGuard](https://github.com/novelKR/DevGuard) is an independent resource authority for development workloads, licensed under Apache-2.0 like CodeSpace. Its approved integration path adds a shared admission and accounting layer while CodeSpace keeps process ownership, PTY, input/output, permissions, approval holds and workspace coordination. -**Current status:** DevGuard has completed DG-1 in its independent repository ([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [milestone ledger](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). On top of the DG-0 contracts it provides native macOS host evidence, an authenticated daemon installed as the current user's LaunchAgent, the fenced launch helper and reconciliation, the `devguard` command-line owner with Cargo adapters, parent leases for candidate tests, and upgrade and repair. Its macOS SLO is qualified for release `0.1.0-5daee5d-b3fa569e` (a release ID, not a Git tag) on the measured host and policy; Linux enforcement is not qualified. CodeSpace consumption (CS-RG) is in progress and not qualified. CSRG-U1 added an opt-in, status-only connection: a gateway built with the `devguard` feature and started with `--devguard status` reports DevGuard's status in `workspace_info`, through DevGuard's client pinned at [`6e7e065`](upstream-lock.md#devguard-client-pin). With CSRG-U2, the process that owns CodeSpace's executions registers itself with DevGuard (`--devguard register`), and a workspace's `resources` setting can require resource participation. With CSRG-U3, that owner takes a process slot and has DevGuard admit each such execution before anything is created. With CSRG-U4 and a configured launch helper (`--devguard-launch-helper`), it then launches the execution once through DevGuard's launch helper, which DevGuard authorizes after binding and policing its scope; without a helper it cancels the unstarted attempt and refuses the execution. Nothing runs without DevGuard. With CSRG-U5, each process has one reaper; DevGuard observes a governed execution's scope before its root is reaped, on a pipe through CodeSpace's own reaper and on a PTY through the pinned Codex PTY's waiter with the [recorded patch](upstream-lock.md#codex-downstream-patch); killing a governed process also terminates its scope; and an approval whose governed execution provably never started can be resumed again. Qualification is a later unit (CSRG-U6). The qualified release remains a candidate for execution, not a selected pin; the client pin is a source pin of the client crates. +**Current status:** DevGuard has completed DG-1 in its independent repository ([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [milestone ledger](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). On top of the DG-0 contracts it provides native macOS host evidence, an authenticated daemon installed as the current user's LaunchAgent, the fenced launch helper and reconciliation, the `devguard` command-line owner with Cargo adapters, parent leases for candidate tests, and upgrade and repair. Its macOS SLO is qualified for release `0.1.0-5daee5d-b3fa569e` (a release ID, not a Git tag) on the measured host and policy; Linux enforcement is not qualified. CodeSpace consumption (CS-RG) is in progress and not qualified. CSRG-U1 added an opt-in, status-only connection: a gateway built with the `devguard` feature and started with `--devguard status` reports DevGuard's status in `workspace_info`, through DevGuard's client pinned at [`6e7e065`](upstream-lock.md#devguard-client-pin). With CSRG-U2, the process that owns CodeSpace's executions registers itself with DevGuard (`--devguard register`), and a workspace's `resources` setting can require resource participation. With CSRG-U3, that owner takes a process slot and has DevGuard admit each such execution before anything is created. With CSRG-U4 and a configured launch helper (`--devguard-launch-helper`), it then launches the execution once through DevGuard's launch helper, which DevGuard authorizes after binding and policing its scope; without a helper it cancels the unstarted attempt and refuses the execution. Nothing runs without DevGuard. With CSRG-U5, each process has one reaper; DevGuard observes a governed execution's scope before its root is reaped, on a pipe through CodeSpace's own reaper and on a PTY through the pinned Codex PTY's waiter with the [recorded patch](upstream-lock.md#codex-downstream-patch); killing a governed process also terminates its scope; and an approval whose governed execution provably never started can be resumed again. Qualification is CSRG-U6: its protocol and harness are in [CS-RG qualification](cs-rg-qualification.md), and nothing is qualified yet. The qualified release remains a candidate for execution, not a selected pin; the client pin is a source pin of the client crates. DevGuard [design revision 1](https://github.com/novelKR/DevGuard/blob/d4981b4c241cff42687f5c2c681b583c7847776e/docs/design-revision-1.md), merged in [DevGuard PR #8](https://github.com/novelKR/DevGuard/pull/8) as `d4981b4c241cff42687f5c2c681b583c7847776e`, revised the CS-RG execution layer and work order. It changes plans and design decisions that bind later implementation. It does not change current CodeSpace behavior or any implementation or qualification status. @@ -267,7 +267,7 @@ Design revision 1 plans CS-RG as 10 work units in six logical PR groups. The IDs | CSRG-P4 | C07, C09 | Parity and fault tests across modes, then the backend convergence decision | | CSRG-P5 | C08 | Qualification of the head that C09 leaves | -CSRG-C09 is a mandatory decision before final qualification. It either integrates the legacy `off` backends and removes the replaced code, branches, fixtures and dependencies, or keeps a limited compatibility backend with a recorded reason, scope, duplicated parts, unsupported capabilities, revisit point and removal criteria. A document stating "reviewed" does not complete it. If C09 changes code, the affected C07 parity tests run again, and CSRG-C08 qualifies that resulting head. +CSRG-C09 is a mandatory decision before final qualification. It either integrates the legacy `off` backends and removes the replaced code, branches, fixtures and dependencies, or keeps a limited compatibility backend with a recorded reason, scope, duplicated parts, unsupported capabilities, revisit point and removal criteria. A document stating "reviewed" does not complete it. If C09 changes code, the affected C07 parity tests run again, and CSRG-C08 qualifies that resulting head. CS-RG keeps `off` as the compatibility path without a code change; the record is in [Legacy `off` path decision](cs-rg-qualification.md#legacy-off-path). This counterpart documentation, listed as CSP-D04 in DevGuard's [PR delivery plan](https://github.com/novelKR/DevGuard/blob/d4981b4c241cff42687f5c2c681b583c7847776e/docs/planning/pr-delivery.md), is a delivery condition before CSRG-P0 starts. The planned structure is in [architecture](architecture.md), its execution contracts in [execution contracts](execution-substrate.md), the reuse decisions in [Codex reuse](codex-reuse.md) and the dependency and CI boundaries in [upstream updates](upstream-update.md). diff --git a/docs/ko/cs-rg-qualification.md b/docs/ko/cs-rg-qualification.md new file mode 100644 index 0000000..1ae1720 --- /dev/null +++ b/docs/ko/cs-rg-qualification.md @@ -0,0 +1,226 @@ + + +# CS-RG qualification ์ ˆ์ฐจ + +[English](../cs-rg-qualification.md) | [ํ•œ๊ตญ์–ด](cs-rg-qualification.md) + +์ด ๋ฌธ์„œ๋Š” CodeSpace๊ฐ€ DevGuard๋ฅผ ์“ฐ๋Š” ๋ฐฉ์‹(CS-RG)์„ qualificationํ•˜๋Š” ์ ˆ์ฐจ์ž…๋‹ˆ๋‹ค. ์–ด๋–ค ์กฐํ•ฉ์„ qualificationํ•˜๋Š”์ง€, ์–ด๋–ค ๋ชจ๋“œ๋ฅผ ์ง€์›ํ•œ๋‹ค๊ณ  ์ฃผ์žฅํ•˜๋Š”์ง€, ๊ฐ ์ฃผ์žฅ์„ ์–ด๋–ป๊ฒŒ ์‹œํ—˜ํ•˜๊ณ  ์ธก์ •ํ•˜๋Š”์ง€, ๋ฌด์—‡์ด ํ†ต๊ณผ์ธ์ง€, ์‹คํ–‰์ด ์–ด๋–ค ์ฆ๊ฑฐ๋ฅผ ๋‚จ๊ธฐ๋Š”์ง€๋ฅผ ์ •ํ•ฉ๋‹ˆ๋‹ค. ์ด ์ ˆ์ฐจ๋Š” ํ•˜๋„ค์Šค [`scripts/qualify-devguard.py`](../../scripts/qualify-devguard.py)์™€ ํ•จ๊ป˜, ์ฒซ qualification ์‹คํ–‰ ์ „์— ๊ณ ์ •ํ•ฉ๋‹ˆ๋‹ค. ์ฒซ qualification ์‹คํ–‰ ๋’ค์— ์ ˆ์ฐจ๋‚˜ ํ•˜๋„ค์Šค๋ฅผ ๋ฐ”๊พธ๋ฉด ์ƒˆ ์ ˆ์ฐจ๊ฐ€ ๋ฉ๋‹ˆ๋‹ค. ๋‹ค๋ฅธ ๋ณ€๊ฒฝ์ฒ˜๋Ÿผ ๊ฒ€ํ† ํ•˜๊ณ , ์˜ํ–ฅ๋ฐ›๋Š” ์‹คํ–‰์€ ๋‹ค์‹œ ํ•ฉ๋‹ˆ๋‹ค. ๊ฐ ๋‹จ์œ„๊ฐ€ ๋ฌด์—‡์„ ๋งŒ๋“ค์—ˆ๋Š”์ง€๋Š” [๊ฒฐํ•ฉ ๋กœ๋“œ๋งต](devguard-integration.md)์— ์žˆ๊ณ , ์ด ๋ฌธ์„œ๋Š” ๊ทธ ๊ฒฐ๊ณผ๊ฐ€ qualification์„ ํ†ต๊ณผํ•˜๋Š”์ง€๋ฅผ ํŒ๋‹จํ•ฉ๋‹ˆ๋‹ค. + +**ํ˜„์žฌ ์ƒํƒœ:** ์ ˆ์ฐจ์™€ ํ•˜๋„ค์Šค๋งŒ ์žˆ์Šต๋‹ˆ๋‹ค. qualification ์‹คํ–‰์€ ์•„์ง ํ•˜์ง€ ์•Š์•˜๊ณ , ์ด ๋ฌธ์„œ์˜ ์–ด๋–ค ๋‚ด์šฉ๋„ qualification ๊ฒฐ๊ณผ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ์ด ์ ˆ์ฐจ๋กœ ํ•œ ์‹คํ–‰์ด ํ†ต๊ณผํ•˜๊ณ  ๊ทธ ์ฆ๊ฑฐ๋ฅผ ๊ฒ€ํ† ํ•  ๋•Œ๊นŒ์ง€ CS-RG๋Š” ์ง„ํ–‰ ์ค‘์ด๋ฉฐ qualification์„ ๋งˆ์น˜์ง€ ์•Š์€ ์ƒํƒœ์ž…๋‹ˆ๋‹ค. + + + +## ํ›„๋ณด ์กฐํ•ฉ + +ํ•œ ๋ฒˆ์˜ ์‹คํ–‰์€ ์ •ํ™•ํžˆ ํ•œ ์กฐํ•ฉ์„ qualificationํ•ฉ๋‹ˆ๋‹ค. ์กฐํ•ฉ์€ ํ•˜๋„ค์Šค์˜ `candidate` ์Šค์œ„ํŠธ๊ฐ€ `candidate.json`์— ๊ธฐ๋กํ•˜๋ฉฐ, ์†์œผ๋กœ ์ ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + +| ๊ตฌ์„ฑ ์š”์†Œ | ๊ธฐ๋กํ•˜๋Š” ๋‚ด์šฉ | +| --- | --- | +| CodeSpace | commit, Codex submodule ๋ฐ–์˜ ์ถ”์  ํŒŒ์ผ์ด ๊ทธ commit๊ณผ ๋‹ค๋ฅธ์ง€ ์—ฌ๋ถ€, `docs/`ยท`docs-site/`ยท`*.md` ํŒŒ์ผ์„ ๋บ€ tree์˜ SHA-256(Codex gitlink ํฌํ•จ) | +| DevGuard client | `Cargo.lock`์˜ `devguard-*` ํ•ญ๋ชฉ([client pin](upstream-lock.md#devguard-client-pin)) | +| Codex | gitlink์™€ `scripts/codex_patch.py check` ๊ฒฐ๊ณผ([๊ธฐ๋ก๋œ ํŒจ์น˜](upstream-lock.md#codex-downstream-patch)) | +| ๋ฐ”์ด๋„ˆ๋ฆฌ | `codespace-mcp`(`-p codespace-server --features devguard`๋กœ ๋นŒ๋“œ)์™€ worker `codespace-codex-runtime`(`--features devguard`)์˜ SHA-256๊ณผ ํฌ๊ธฐ. fixture, helper, `devguard-qualify` ๋ฐ”์ด๋„ˆ๋ฆฌ๋ฅผ ์ฃผ๋ฉด ๊ทธ๊ฒƒ๋„ ๊ธฐ๋ก | +| DevGuard release | ์„ค์น˜๋œ release ID, ๊ทธ `MANIFEST.json`์˜ SHA-256, ์„ค์น˜๋œ ๊ฐ ๋ฐ”์ด๋„ˆ๋ฆฌ๊ฐ€ manifest์™€ ๋งž๋Š”์ง€, ์„œ๋น„์Šค ์ƒํƒœ | +| ์ •์ฑ… | `host.toml`์˜ SHA-256, `devguard doctor --require admission,registration,macos-cooperative`(๊ตฌ์„ฑ ์ง€๋ฌธ, capability, protocol), `codespace` ์†Œ๋น„์ž์˜ roleยทgenerationยท์ธ์Šคํ„ด์Šค ํ•œ๋„ยทcontrol ์˜ˆ์•ฝ. ์†Œ๋น„์ž credential digest๋Š” ๊ธฐ๋ก์— ์ฝ์–ด ๋“ค์ด์ง€ ์•Š์Œ | +| DG-1 qualification | ๊ทธ release์— ๋Œ€ํ•œ DevGuard์˜ qualification ๊ธฐ๋ก์ด ์žˆ์œผ๋ฉด ๊ทธ SHA-256, ํŒ์ •, ๊ฐ™์€ `host.toml`ยท์ง€๋ฌธยทcapability์—์„œ ์ธก์ •ํ–ˆ๋Š”์ง€ ์—ฌ๋ถ€ | +| ๋„๊ตฌ์™€ ํ˜ธ์ŠคํŠธ | `rustc -vV`, `cargo -V`, Python, Xcode, macOS ๋ฒ„์ „, ๋ชจ๋ธ, ๋…ผ๋ฆฌ CPU, ๋ฉ”๋ชจ๋ฆฌ, ๋ถ€ํŒ… ์‹œ๊ฐ, ์„œ๋น„์Šค์˜ work capacity์™€ pressure ์ƒํƒœ | + +`candidate`๋Š” Codex submodule ๋ฐ–์— ์ปค๋ฐ‹ํ•˜์ง€ ์•Š์€ ๋ณ€๊ฒฝ์ด ์—†๊ณ , Codex ํ™•์ธ์ด ํ†ต๊ณผํ•˜๊ณ (submodule์ด ๊ณ ์ • commit์— ๊ธฐ๋ก๋œ ํŒจ์น˜๋งŒ ์ •ํ™•ํžˆ ์ ์šฉํ•œ ์ƒํƒœ), ์„œ๋น„์Šค๊ฐ€ ๊ทธ release๋ฅผ ์‹คํ–‰ํ•˜๊ณ , ์„ค์น˜๋œ ๋ฐ”์ด๋„ˆ๋ฆฌ๊ฐ€ ๋ชจ๋‘ manifest์™€ ๋งž๊ณ , `doctor`๊ฐ€ ๋งŒ์กฑํ•˜๊ณ , `host.toml`์— `control_service` role์˜ `codespace` ์†Œ๋น„์ž๊ฐ€ ์žˆ๊ณ , `PATH`์—์„œ Rust 1.95.0์ด ๋จผ์ € ์žกํž ๋•Œ ํ†ต๊ณผํ•ฉ๋‹ˆ๋‹ค. ์ตœ์ข… ๋ณด๊ณ ์„œ์—๋Š” ๊ฐ™์€ ์ •์ฑ…์—์„œ ๊ฐ™์€ release๋ฅผ ์ธก์ •ํ•œ DG-1 qualification๋„ ์žˆ์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. `candidate`๋Š” ๊ทธ ๊ธฐ๋ก์„ ์š”๊ตฌํ•˜์ง€ ์•Š๊ณ  ๊ธฐ๋ก๋งŒ ํ•˜๋ฏ€๋กœ, ๊ทธ ๊ธฐ๋ก์ด ์ƒ๊ธฐ๊ธฐ ์ „์—๋„ ํ•˜๋„ค์Šค๋ฅผ ์‹คํ–‰ํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. + +์ฝ”๋“œ, ๋ฐ”์ด๋„ˆ๋ฆฌ, release, ์ •์ฑ…, Codex, ๋„๊ตฌ ๊ฐ€์šด๋ฐ ๋ฌด์—‡์ด๋“  ๋ฐ”๋€Œ๋ฉด ์ƒˆ ํ›„๋ณด๊ฐ€ ๋˜๊ณ , ์˜ํ–ฅ๋ฐ›๋Š” ์‹คํ–‰์„ ๋‹ค์‹œ ํ•ฉ๋‹ˆ๋‹ค. ๋ฌธ์„œ๋งŒ ๋ฐ”๊พผ commit์€ ํ›„๋ณด๋ฅผ ์œ ์ง€ํ•˜๋ฉฐ, code tree hash๊ฐ€ ์ด๋ฅผ ๋ณด์—ฌ ์ค๋‹ˆ๋‹ค. + + + +## ์ง€์› ๋ชจ๋“œ + +์ฃผ์žฅํ•˜๋Š” ์นธ์€ ๊ฐ๊ฐ ๋”ฐ๋กœ ์‹œํ—˜ํ•ฉ๋‹ˆ๋‹ค. ์–ด๋–ค ์นธ๋„ ๋‹ค๋ฅธ ์นธ์—์„œ ์ถ”๋ก ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + +| ํ”Œ๋žซํผ | runner | I/O | ์ฐธ์—ฌ | ์ฃผ์žฅ | ๊ทผ๊ฑฐ | +| --- | --- | --- | --- | --- | --- | +| macOS, ์‹ค์ œ ํ˜ธ์ŠคํŠธ ์ฆ๊ฑฐ๊ฐ€ ์žˆ๋Š” ์„ค์น˜ ์„œ๋น„์Šค | `in-process` | pipe, PTY | `required` | qualification | ๊ธฐ๋Šฅ ํ–‰๋ ฌ, ์žฅ์•  ์‚ฌ๋ก€, ์ œ์–ด ๊ฒฝ๋กœ ์ธก์ • | +| macOS | `--runtime-bin`์„ ์“ด `uds`(worker๊ฐ€ ์‹คํ–‰์„ ์†Œ์œ ) | pipe, PTY | `required` | qualification | ์œ„์™€ ๊ฐ™์Œ. worker๊ฐ€ ๋“ฑ๋ก๋จ | +| macOS | `--runner-socket`์„ ์“ด `uds` | pipe, PTY | `required` | ์ง€์›ํ•˜์ง€ ์•Š์Œ | attempt๊ฐ€ ์ƒ๊ธฐ๊ธฐ ์ „์— `RESOURCE_POLICY_UNSUPPORTED`๋กœ ๊ฑฐ๋ถ€ | +| macOS | ๋ชจ๋‘ | pipe, PTY | `--devguard status`์—์„œ์˜ `required` | ์ง€์›ํ•˜์ง€ ์•Š์Œ | attempt๊ฐ€ ์ƒ๊ธฐ๊ธฐ ์ „์— `RESOURCE_POLICY_UNSUPPORTED`๋กœ ๊ฑฐ๋ถ€ | +| macOS | ์ง€์› ๋ชจ๋“œ | pipe, PTY | ์“ธ ์ˆ˜ ์žˆ๋Š” launch helper๊ฐ€ ์—†๋Š” `required` | ์ง€์›ํ•˜์ง€ ์•Š์Œ | ์‹คํ–‰ ํ—ˆ์šฉ ๋’ค attempt๋ฅผ ์ทจ์†Œํ•˜๊ณ  `MANAGED_LAUNCH_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€ | +| macOS | ์ง€์› ๋ชจ๋“œ | pipe, PTY | gateway ํ•˜๋‚˜์— `off`์™€ `required` workspace๋ฅผ ํ•จ๊ป˜ | qualification | ๊ด€๋ฆฌ ๋Œ€์ƒ๊ณผ ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” ์ž์‹์ด ๋™์‹œ์— ์‹คํ–‰๋˜๊ณ , ๊ฐ ์ž์‹์€ ํ‘œ์ค€ descriptor๋งŒ ๊ฐ€์ง | +| macOS, Linux | ๋ชจ๋‘ | pipe, PTY | `off` | ๊ธฐ์กด ๋™์ž‘ | `devguard` feature๋ฅผ ์ผœ๊ณ  ๋ˆ [ํšŒ๊ท€ ๋‹จ๊ณ„](#regressions) | +| Linux | ๋ชจ๋‘ | pipe, PTY | `required` | DG-LINUX ์ „๊นŒ์ง€ ์ง€์›ํ•˜์ง€ ์•Š์Œ | ์•„๋ฌด๊ฒƒ๋„ ์‹คํ–‰ํ•˜๊ธฐ ์ „์— ๊ฑฐ๋ถ€. `--devguard register`์—์„œ๋Š” ์‹ค์ œ ํ˜ธ์ŠคํŠธ ์ฆ๊ฑฐ๊ฐ€ ์—†์–ด ์†Œ์œ ์ž๊ฐ€ ๋“ฑ๋กํ•˜์ง€ ๋ชปํ•˜๋ฏ€๋กœ `RESOURCE_AUTHORITY_UNAVAILABLE`, ๊ทธ ๋ฐ–์—๋Š” `RESOURCE_POLICY_UNSUPPORTED`. Linux CI๊ฐ€ ๋ณด์—ฌ ์คŒ | + +`devguard` feature ์—†์ด ๋นŒ๋“œํ•œ gateway์—๋Š” DevGuard ์˜ต์…˜์ด ์—†์Šต๋‹ˆ๋‹ค. ๊ทธ๋Ÿฐ gateway์—์„œ `required` workspace๋Š” `--devguard status`์ผ ๋•Œ์ฒ˜๋Ÿผ ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค. + + + +## ํ•˜๋„ค์Šค ์‹คํ–‰ + +ํ•˜๋„ค์Šค๋Š” release๋ฅผ ์„ค์น˜ํ•˜๊ณ  `codespace` ์†Œ๋น„์ž๋ฅผ ๋“ฑ๋กํ•œ Mac, ๊ณง qualification ํ˜ธ์ŠคํŠธ์—์„œ ์‹œ์Šคํ…œ Python(3.9 ์ด์ƒ)์œผ๋กœ ์‹คํ–‰ํ•˜๋ฉฐ ์™ธ๋ถ€ ํŒจํ‚ค์ง€๋ฅผ ์“ฐ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ํ•˜๋„ค์Šค๊ฐ€ ์‹œ์ž‘ํ•˜๋Š” ํ”„๋กœ์„ธ์Šค๋Š” ๋ชจ๋‘ ์ตœ์†Œ ํ™˜๊ฒฝ(`HOME`, `PATH=/usr/bin:/bin:/usr/sbin:/sbin`, `TMPDIR`, `LANG=C`์™€ ํ•˜๋„ค์Šค๊ฐ€ ์ง€์ •ํ•˜๋Š” `CODESPACE_*` ์„ค์ •)๋งŒ ๋ฐ›์œผ๋ฉฐ, ํ•˜๋„ค์Šค ์ž์‹ ๋„ ๊ฐ™์€ ๋ฐฉ์‹์œผ๋กœ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. + +```bash +env -i HOME="$HOME" TMPDIR="$TMPDIR" PATH="/bin:/usr/bin:/bin:/usr/sbin:/sbin" \ + /usr/bin/python3 scripts/qualify-devguard.py matrix --out evidence/<์ƒˆ ๋””๋ ‰ํ„ฐ๋ฆฌ> \ + --mcp-bin --runtime-bin --release +``` + +| ์Šค์œ„ํŠธ | ๋Œ€์ƒ | ์ถ”๊ฐ€ ์ธ์ž | +| --- | --- | --- | +| `candidate` | ์„ค์น˜๋œ ์„œ๋น„์Šค | `--release`. ๊ธฐ๋กํ•˜๋ ค๋ฉด `--fixture-bin`, `--fixture-helper`, `--qualify-bin`๋„ | +| `matrix` | ์„ค์น˜๋œ ์„œ๋น„์Šค | `--release` | +| `faults` | DevGuard์˜ fixture authority | `--fixture-bin`(`crates/devguard`์˜ ์˜ˆ์ œ `fixture_authority`), `--fixture-helper`(client pin์—์„œ ๋นŒ๋“œํ•œ `devguard-launch`) | +| `authority` | DevGuard์˜ fixture authority | `faults`์™€ ๊ฐ™์Œ | +| `control` | DG-1 ๋ถ€ํ•˜ ์•„๋ž˜์˜ ์„ค์น˜๋œ ์„œ๋น„์Šค | `--release`, `--work`(์ƒˆ ๋””๋ ‰ํ„ฐ๋ฆฌ), `--qualify-bin`(DevGuard์˜ `devguard-qualify`), `--devguard-checkout`(release์˜ ์†Œ์Šค commit์ด ์žˆ๋Š” DevGuard checkout), `--load cold` ๋˜๋Š” `warm`, `--rehearsal`, `--repetitions` | +| `verify` | ๋ด‰์ธ๋œ ์ฆ๊ฑฐ ๋””๋ ‰ํ„ฐ๋ฆฌ | ๊ทธ ๋””๋ ‰ํ„ฐ๋ฆฌ | + +`--modes`๋กœ ์Šค์œ„ํŠธ๋ฅผ `in-process`๋‚˜ `uds` ํ•˜๋‚˜๋กœ ์ค„์ผ ์ˆ˜ ์žˆ์œผ๋ฉฐ, ๊ธฐ๋ณธ์€ ๋‘˜ ๋‹ค์ž…๋‹ˆ๋‹ค. ์„ค์น˜๋œ ์„œ๋น„์Šค๋ฅผ ์“ฐ๋Š” ์Šค์œ„ํŠธ๋Š” ๊ธฐ๋ณธ์œผ๋กœ ์„œ๋น„์Šค์˜ socket, journal, ๋กœ๊ทธ, `host.toml`, ์†Œ๋น„์ž credential ํŒŒ์ผ์„ ์”๋‹ˆ๋‹ค. credential ํŒŒ์ผ์€ ๊ฒฝ๋กœ๋งŒ gateway์— ๋„˜๊ธฐ๊ณ  ํ•˜๋„ค์Šค๋Š” ์ฝ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ข…๋ฃŒ ์ƒํƒœ๋Š” 0 ํ†ต๊ณผ ๋˜๋Š” qualification, 1 ์‹คํŒจ, 2 ๋ฏธ์™„๋ฃŒ ๋˜๋Š” ํŒ์ • ๋ถˆ๊ฐ€, 3 ํ•˜๋„ค์Šค ์‹คํŒจ, 130 ์ค‘๋‹จ์ž…๋‹ˆ๋‹ค. `verify`๋Š” ์‹คํ–‰ ๊ธฐ๋ก์ด ๋งž์œผ๋ฉด 0, ์•„๋‹ˆ๋ฉด 1๋กœ ๋๋‚ฉ๋‹ˆ๋‹ค. + +ํ•˜๋„ค์Šค๋Š” ๋‹ค๋ฅธ MCP client์™€ ๋งˆ์ฐฌ๊ฐ€์ง€๋กœ ํ‘œ์ค€ ์ž…์ถœ๋ ฅ์œผ๋กœ `codespace-mcp`์™€ ํ†ต์‹ ํ•˜๋ฉฐ(protocol `2025-11-25`), ์ž์ฒด์˜ ์ž‘์€ client๋ฅผ ์”๋‹ˆ๋‹ค. ๋”ฐ๋ผ์„œ ์‹œ๊ฐ„ ๊ฐ’์€ ํ•˜๋„ค์Šค์—์„œ ์žฐ ๋„๊ตฌ ํ˜ธ์ถœ์˜ ๋กœ์ปฌ ์™•๋ณต ์‹œ๊ฐ„์ด๊ณ  ๋„คํŠธ์›Œํฌ ์‹œ๊ฐ„์€ ๋“ค์–ด๊ฐ€์ง€ ์•Š์Šต๋‹ˆ๋‹ค. DevGuard์˜ journal์€ DevGuard ์ž์‹ ์˜ ์ธก์ •์ฒ˜๋Ÿผ ์ฝ๊ธฐ ์ „์šฉ์œผ๋กœ ์ฝ๊ณ , ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์˜ attempt ID์™€ scope root๋Š” gateway ๋กœ๊ทธ์—์„œ ๊ฐ€์ ธ์˜ต๋‹ˆ๋‹ค. + + + +## ๊ธฐ๋Šฅ ํ–‰๋ ฌ + +`matrix`๋Š” ์„œ๋น„์Šค๊ฐ€ release๋ฅผ ์‹คํ–‰ํ•˜๊ณ  ์ž์›์„ ์ฐจ์ง€ํ•œ ์ž‘์—…์ด ์—†์Œ์„ ํ™•์ธํ•œ ๋’ค, ์„ค์น˜๋œ ์„œ๋น„์Šค๋ฅผ ๋Œ€์ƒ์œผ๋กœ runner ๋ชจ๋“œ๋งˆ๋‹ค ํ•œ ๋ฒˆ ์‹คํ–‰ํ•ฉ๋‹ˆ๋‹ค. ๊ด€๋ฆฌ ๋Œ€์ƒ workspace๋Š” 50 mCPU, 64 MiB, 8 tasks๋ฅผ ์š”์ฒญํ•˜๊ณ , gateway์˜ ํ”„๋กœ์„ธ์Šค timeout์€ 15์ดˆ์ž…๋‹ˆ๋‹ค. ์ž์†์„ ๋‚จ๊ธฐ๊ฑฐ๋‚˜, ๊ทธ๋ฃน์„ ๋ฒ—์–ด๋‚˜๊ฑฐ๋‚˜, ์„œ๋น„์Šค๋ฅผ ์žฌ์‹œ์ž‘ํ•˜๋Š” ๋ช…๋ น์€ ์‹คํ–‰ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๊ทธ๋ž˜์„œ ์†Œ์œ ์ž์˜ ์„œ๋น„์Šค์— suspect๋กœ ๋‚จ๋Š” attempt๊ฐ€ ์ƒ๊ธฐ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + +| ์‹œ๋‚˜๋ฆฌ์˜ค | pipe์™€ PTY | ํ†ต๊ณผ ์กฐ๊ฑด | +| --- | --- | --- | +| registration | | `workspace_info`๊ฐ€ ์ฐธ์—ฌ `registration`, authority ์‚ฌ์šฉ ๊ฐ€๋Šฅ, ์†Œ์œ ์ž(`in_process` ๋˜๋Š” `worker`) ๋“ฑ๋ก, ์‹คํ–‰ ๊ด€๋ฆฌ ์ค‘์„ ๋ณด๊ณ ํ•˜๊ณ , DevGuard์˜ ํ™œ์„ฑ `codespace` ์ธ์Šคํ„ด์Šค ํ•˜๋‚˜๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค์ž„ | +| exits | ๊ฐ๊ฐ | exit 0๊ณผ exit 3์ด ๊ทธ ์ฝ”๋“œ์˜ exited๋กœ ๋ณด๊ณ ๋จ. SIGTERM์œผ๋กœ ์ฃฝ์œผ๋ฉด pipe์—์„œ๋Š” signal 15์˜ `signaled`, PTY์—์„œ๋Š” ๊ณ ์ •๋œ Codex PTY๊ฐ€ ๋ณด๊ณ ํ•˜๋Š” ๋Œ€๋กœ ์ฝ”๋“œ 1์˜ exited([ํ”„๋กœ์„ธ์Šค ์ƒํƒœ](agent-integration.md)) | +| io | ๊ฐ๊ฐ | pipe์—์„œ๋Š” `write_stdin`์ด `cat`์— ๋‹ฟ๊ณ  ์ถœ๋ ฅ์ด ๋Œ์•„์˜ค๋ฉฐ, `process_resize`๊ฐ€ `PROCESS_NOT_TTY`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , ์ข…๋ฃŒ๊ฐ€ ๋ณด๊ณ ๋จ. PTY์—์„œ๋Š” ํ„ฐ๋ฏธ๋„์ด 24ร—80์œผ๋กœ ์‹œ์ž‘ํ•˜๊ณ , 40ร—100 ํฌ๊ธฐ ์กฐ์ •์— ๋‹ตํ•˜๋ฉฐ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์ƒˆ ํฌ๊ธฐ๋ฅผ ๋ณด๊ณ , stdin์ด ๋‹ฟ์Œ | +| termination | ๊ฐ๊ฐ | `terminate_process`๊ฐ€ ๋‹ตํ•˜๊ณ , ํ”„๋กœ์„ธ์Šค๊ฐ€ terminated๋กœ ๋ณด๊ณ ๋˜๋ฉฐ, scope์˜ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน์ด ์‚ฌ๋ผ์ง | +| timeout | ๊ฐ๊ฐ | `sleep 600`์ด `timeout`์œผ๋กœ ๋ณด๊ณ ๋จ | +| exec-failed | ๊ฐ๊ฐ | ์ธํ„ฐํ”„๋ฆฌํ„ฐ๊ฐ€ ์—†๊ฑฐ๋‚˜ ์‹คํ–‰ํ•  ์ˆ˜ ์—†๋Š” ์Šคํฌ๋ฆฝํŠธ๊ฐ€ errno 2๋‚˜ 13๊ณผ ํ•จ๊ป˜ `PROCESS_SPAWN_FAILED`๋กœ ๊ฑฐ๋ถ€๋จ. helper๊ฐ€ grant๋ฅผ ๋ฐ›์•˜๊ณ  scope๊ฐ€ ๋๋‚˜์ž DevGuard๊ฐ€ ํ•ด์ œํ•จ | +| shortage | ๊ฐ๊ฐ | work capacity๋ณด๋‹ค ๋งŽ์€ CPU๋ฅผ ์š”์ฒญํ•˜๋Š” workspace์— ์„ธ ๋ฒˆ ํ˜ธ์ถœํ•˜๋ฉด ๋งค๋ฒˆ ์ƒˆ attempt๊ฐ€ ๊ฑฐ๋ถ€(denied)๋˜์–ด `RESOURCE_UNAVAILABLE`์ด ๋˜๊ณ , ์ž์›์„ ์ฐจ์ง€ํ•˜์ง€ ์•Š์œผ๋ฉฐ, `WORKSPACE_BUSY`๋Š” ๋‚˜์˜ค์ง€ ์•Š์Œ | +| unsupported-minimum | ๊ฐ๊ฐ | ์ตœ์†Œ ์š”๊ตฌ๋กœ kernel์˜ task ๊ฐ•์ œ๋ฅผ ์š”๊ตฌํ•˜๋Š” workspace๊ฐ€ DevGuard์—์„œ ๊ฑฐ๋ถ€๋˜์–ด `RESOURCE_POLICY_UNSUPPORTED`๊ฐ€ ๋จ | +| mixed | ๊ฐ๊ฐ | gateway ํ•˜๋‚˜์—์„œ ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” ์‹คํ–‰๊ณผ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์ด ๋™์‹œ์— ์‹คํ–‰๋˜๊ณ , ๊ฐ ์ž์‹์€ descriptor 0, 1, 2๋งŒ ๊ฐ€์ง | +| no-helper, unusable-helper | | launch helper๊ฐ€ ์—†๊ฑฐ๋‚˜ ์กด์žฌํ•˜์ง€ ์•Š๋Š” helper๋ฅผ ์ค€ gateway๋Š” `MANAGED_LAUNCH_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€ํ•˜๊ณ , DevGuard๋Š” attempt๋ฅผ ์ทจ์†Œ(cancelled)๋กœ ๊ธฐ๋กํ•˜๋ฉฐ, `off` workspace๋Š” ๊ณ„์† ์‹คํ–‰๋จ | +| status-mode | | `--devguard status`์—์„œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ attempt๊ฐ€ ์ƒ๊ธฐ๊ธฐ ์ „์— `RESOURCE_POLICY_UNSUPPORTED`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , `off`๋Š” ๊ณ„์† ์‹คํ–‰๋จ | +| authority-unavailable | | socket์— authority๊ฐ€ ์—†์œผ๋ฉด `RESOURCE_AUTHORITY_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , DevGuard์—๋Š” ๊ทธ attempt๊ฐ€ ์—†์œผ๋ฉฐ, `off`๋Š” ๊ณ„์† ์‹คํ–‰๋จ | +| runner-socket | `uds`๋งŒ | gateway๊ฐ€ ์‹œ์ž‘ํ•˜์ง€ ์•Š์€ worker์—์„œ๋Š” ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์ด attempt๊ฐ€ ์ƒ๊ธฐ๊ธฐ ์ „์— `RESOURCE_POLICY_UNSUPPORTED`๋กœ ๊ฑฐ๋ถ€๋จ | +| settled | | ์†Œ๋น„์ž์˜ ์–ด๋–ค attempt๋„ ์ž์›์„ ์ฐจ์ง€ํ•˜์ง€ ์•Š๊ณ  ํ™œ์„ฑ ์ธ์Šคํ„ด์Šค๋„ ์—†์Œ. ์•„๋‹ˆ๋ฉด ํ–‰๋ ฌ์„ ๋ฉˆ์ถค | + +๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ ๋ชจ๋‘ DevGuard ์ž์‹ ์˜ attempt ๊ธฐ๋ก๊ณผ ๋‹ค์‹œ ๋Œ€์กฐํ•ฉ๋‹ˆ๋‹ค([์ž์› ํšŒ๊ณ„](#accounting)). + + + +## ์žฅ์•  ์‚ฌ๋ก€ + +`faults`๊ฐ€ ๋„ฃ๋Š” ์žฅ์• ๋Š” ์„ค์น˜๋œ ์„œ๋น„์Šค์— suspect๋‚˜ ์ž์›์„ ์ฐจ์ง€ํ•œ ์ž‘์—…์„ ๋‚จ๊ธธ ์ˆ˜ ์žˆ์œผ๋ฏ€๋กœ, DevGuard์˜ fixture authority๋ฅผ ๋Œ€์ƒ์œผ๋กœ ์‹คํ–‰ํ•ฉ๋‹ˆ๋‹ค. fixture authority๋Š” CodeSpace์˜ client pin์— ์žˆ๋Š” DevGuard daemon ์ฝ”๋“œ๋ฅผ ๋ณ„๋„ ์ƒํƒœ ๋””๋ ‰ํ„ฐ๋ฆฌ์—์„œ ์‹คํ–‰ํ•˜๋ฉฐ, `codespace` ์†Œ๋น„์ž๋ฅผ ์Šค์Šค๋กœ ๋“ฑ๋กํ•ฉ๋‹ˆ๋‹ค. ์‚ฌ๋ก€๋งˆ๋‹ค ์ƒˆ fixture authority๋ฅผ ์“ฐ๊ณ , ๊ด€๋ฆฌ ๋Œ€์ƒ workspace๋Š” 50 mCPU, 32 MiB, 4 tasks๋ฅผ ์š”์ฒญํ•ฉ๋‹ˆ๋‹ค. + +| ์‚ฌ๋ก€ | pipe์™€ PTY | ํ†ต๊ณผ ์กฐ๊ฑด | +| --- | --- | --- | +| descendant | ๊ฐ๊ฐ | ์ž์†์„ ๋‚จ๊ธฐ๊ณ  ๋๋‚œ root์˜ attempt๋Š” ์ž์†์ด ์‹คํ–‰๋˜๋Š” ๋™์•ˆ suspect๊ฐ€ ์•„๋‹Œ ์ƒํƒœ๋กœ ์ž์›์„ ์ฐจ์ง€ํ•˜๊ณ , ์ž์†์ด ๋๋‚˜๋ฉด ํ•ด์ œ๋จ. PTY์—์„œ๋Š” ์ข…๋ฃŒ ์ „ํ›„๋กœ reconciler๋ฅผ ๋ฉˆ์ถฐ ๋‘๋ฏ€๋กœ reap ์ „ ๊ด€์ฐฐ๋งŒ์ด ์ž์†์„ ์ฑ„ํƒํ•  ์ˆ˜ ์žˆ์Œ | +| escape | ๊ฐ๊ฐ | `setsid`๋กœ scope์˜ ๊ทธ๋ฃน์„ ๋ฒ—์–ด๋‚œ ์ž์†์ด ์žˆ์œผ๋ฉด ๋ชจ๋“  ๊ฒƒ์ด ๋๋‚œ ๋’ค์—๋„ attempt๊ฐ€ suspect๋กœ ์ž์›์„ ์ฐจ์ง€ํ•จ | +| stalled-observer | ๊ฐ๊ฐ | reconciler๋ฅผ ๋ฉˆ์ถ”๊ณ  authority๋ฅผ ๊ด€์ฐฐ ์ƒํ•œ๋ณด๋‹ค ์˜ค๋ž˜ ์žก์•„ ๋‘๋ฉด, CodeSpace๋Š” ์žก์•„ ๋‘” ์‹œ๊ฐ„์ด ๋๋‚˜๊ธฐ ์ „์— root๋ฅผ reapํ•˜๊ณ , ์ฆ๋ช…๋˜์ง€ ์•Š์€ ์ž์† ๋•Œ๋ฌธ์— attempt๋Š” ๊ทธ ์ž์†์ด ๋๋‚œ ๋’ค์—๋„ suspect๋กœ ๋‚จ์Œ | +| authority-lost | | pipe์™€ PTY ์‹คํ–‰ ์ค‘์— authority๋ฅผ ์ฃฝ์—ฌ๋„ ๋‘˜ ๋‹ค ๊ณ„์† `running`์œผ๋กœ ๋‹ตํ•˜๊ณ  4์ดˆ ์•ˆ์— ์ข…๋ฃŒ๋˜๋ฉฐ, ์ƒˆ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ DevGuard์— ์•„๋ฌด๊ฒƒ๋„ ๋‚จ๊ธฐ์ง€ ์•Š๊ณ  `RESOURCE_AUTHORITY_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , `off`๋Š” ๊ณ„์† ์‹คํ–‰๋จ | +| authority-restart | | ์Šน์ธ ํ›„ ์žฌ๊ฐœํ•œ ์‹คํ–‰ ์ค‘์— authority๋ฅผ ์žฌ์‹œ์ž‘ํ•ด๋„ ํ”„๋กœ์„ธ์Šค๋Š” ๊ณ„์† CodeSpace์˜ ๊ฒƒ์ด๊ณ  ๋๊นŒ์ง€ ์‹คํ–‰๋˜๋ฉฐ, ๊ทธ attempt๋Š” suspect๋กœ ์ž์›์„ ์ฐจ์ง€ํ•˜๊ณ , ์Šน์ธ์„ ๋‹ค์‹œ ์žฌ๊ฐœํ•˜๋ฉด ๊ฐ™์€ ํ”„๋กœ์„ธ์Šค๋ฅผ ๋Œ๋ ค์ฃผ๋ฉฐ ์‹คํ–‰์€ ํ•œ ๋ฒˆ๋ฟ์ด๊ณ , ์ƒˆ ์‹คํ–‰์€ ํ—ˆ์šฉ๋˜๊ณ  ํ•ด์ œ๋จ | +| authority-stalled | | authority์˜ lock์„ ์†Œ์œ ์ž์˜ deadline๋ณด๋‹ค ์˜ค๋ž˜ ์žก์•„ ๋‘๋ฉด ํ˜ธ์ถœ์€ ์žก์•„ ๋‘” ์‹œ๊ฐ„์ด ๋๋‚˜๊ธฐ ์ „์— `ADMISSION_UNKNOWN`์ด๋‚˜ `RESOURCE_AUTHORITY_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , ๋ช…๋ น์€ ์‹คํ–‰๋˜์ง€ ์•Š์œผ๋ฉฐ, DevGuard๋Š” attempt๋ฅผ helper์— ๋„˜๊ธฐ์ง€ ์•Š๊ณ  ์ž์›์„ ๋‚จ๊ธฐ์ง€ ์•Š์œผ๋ฉฐ, ๋‹ค์Œ ์‹คํ–‰์€ ์ƒˆ attempt๋กœ ์‹คํ–‰๋จ | +| helper-ends | | READY ์ „์— ๋๋‚˜๋Š” helper๋ฅผ ์“ฐ๋ฉด ํ˜ธ์ถœ์€ `PROCESS_SPAWN_FAILED`(ready๋ฅผ ๋ณด๊ณ ํ•˜๊ธฐ ์ „์— ๋๋‚จ)๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , ๋ช…๋ น์€ ์‹คํ–‰๋˜์ง€ ์•Š์œผ๋ฉฐ, DevGuard๋Š” grant๋ฅผ ์‹œ์ž‘ํ•˜์ง€ ์•Š์€ ๊ฒƒ(`no_helper_created`)์œผ๋กœ ํ•ด์ œํ•จ | +| credential | | authority๊ฐ€ ๊ฑฐ๋ถ€ํ•˜๋Š” credential๊ณผ ์—†๋Š” credential ๊ฐ๊ฐ์—์„œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ `credential_refused` ๋˜๋Š” `credential_unavailable`์„ ๋ฐํžˆ๋ฉฐ `RESOURCE_AUTHORITY_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , `off`๋Š” ๊ณ„์† ์‹คํ–‰๋จ | +| terminate-race | | ํ”„๋กœ์„ธ์Šค๊ฐ€ ์Šค์Šค๋กœ ๋๋‚˜๋Š” ์ˆœ๊ฐ„ ๊ทผ์ฒ˜์—์„œ pipe์™€ PTY๋ฅผ ๋ฒˆ๊ฐˆ์•„ 12๋ฒˆ ์ข…๋ฃŒํ•ด๋„ ๊ฐ๊ฐ exited๋‚˜ terminated๋กœ ํ•œ ๋ฒˆ๋งŒ ๋๋‚˜๊ณ , ํ•œ ๋ฒˆ๋งŒ ์‹คํ–‰๋˜๋ฉฐ, ํ•ด์ œ๋จ | +| shutdown | | ๊ด€๋ฆฌ ๋Œ€์ƒ pipe์™€ PTY ์‹คํ–‰ ์ค‘์— gateway์˜ ์ž…๋ ฅ์„ ๋‹ซ์œผ๋ฉด(`uds` ๋ชจ๋“œ์—์„œ๋Š” worker๋„ ํ•จ๊ป˜ ๋๋‚จ) gateway์™€ ์†Œ์œ ์ž๊ฐ€ ์ข…๋ฃŒํ•˜๊ณ , ๋‘ scope๊ฐ€ ๋๋‚˜ ํ•ด์ œ๋˜๋ฉฐ, ์ƒˆ gateway๋Š” ๋“ฑ๋กํ•˜๊ณ  ์‹คํ–‰ํ•˜๋˜ ๋‘˜ ์ค‘ ์–ด๋Š ๊ฒƒ๋„ ๋‹ค์‹œ ์‹คํ–‰ํ•˜์ง€ ์•Š์Œ | +| worker-killed | `uds`๋งŒ | worker๊ฐ€ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์„ ์ค€๋น„ํ•˜๋Š” ์ค‘์— worker๋ฅผ ์ฃฝ์—ฌ๋„ gateway๋Š” ํ˜ธ์ถœ์— ๋‹ตํ•˜๊ณ , ๋ช…๋ น์€ ์‹คํ–‰๋˜์ง€ ์•Š์œผ๋ฉฐ, DevGuard๋Š” attempt๋ฅผ helper์— ๋„˜๊ธฐ์ง€ ์•Š๊ณ  ์ž์›์„ ๋‚จ๊ธฐ์ง€ ์•Š์œผ๋ฉฐ, gateway๋Š” ๊ณ„์† ์‹คํ–‰๋จ | + + + +### ์ž‘์—… ์ง€์‹œ์„œ์˜ ์žฅ์•  ๋ชฉ๋ก๊ณผ์˜ ๋Œ€์‘ + +| ์žฅ์•  | ๋ณด์—ฌ ์ฃผ๋Š” ๊ณณ | +| --- | --- | +| ์‹คํ–‰ ํ—ˆ์šฉ ๊ฑฐ๋ถ€ | ํ–‰๋ ฌ `shortage`, `unsupported-minimum`(์„ค์น˜๋œ ์„œ๋น„์Šค, ๋‘ ๋ชจ๋“œ, pipe์™€ PTY) | +| ์‹คํ–‰ ํ—ˆ์šฉ ์ „ authority ์‚ฌ๋ผ์ง | ํ–‰๋ ฌ `authority-unavailable`, ์žฅ์•  `credential` | +| ์‹œ์ž‘ ๋’ค authority ์‚ฌ๋ผ์ง | ์žฅ์•  `authority-lost`, `authority-restart`, `authority` ์Šค์œ„ํŠธ | +| launch ๋‹ต ์žƒ์Œ | ์„ธ์…˜์ด ๊ธฐ๋‹ค๋ฆฌ๋Š” ์š”์ฒญ์— ๋Œ€ํ•ด์„œ๋Š” ์žฅ์•  `authority-stalled`. ๋“ฑ๋ก์„ ์ง€๋‚œ ์š”์ฒญ์˜ ๋‹ต์„ ์žƒ๋Š” ๊ฒฝ์šฐ๋Š” ์†Œ์œ ์ž ๋ฐ–์—์„œ ๋งŒ๋“ค ์ˆ˜ ์—†์œผ๋ฏ€๋กœ, DevGuard์˜ client ์‹œํ—˜๊ณผ ์Šคํฌ๋ฆฝํŠธ worker๋ฅผ ์“ฐ๋Š” CodeSpace์˜ runnerยทgateway ์‹œํ—˜์ด ๋‹ค๋ฃธ | +| helper ์‹คํŒจ | ํ–‰๋ ฌ `no-helper`, `unusable-helper`, ์žฅ์•  `helper-ends` | +| READY ๋’ค ํ”„๋กœ๊ทธ๋žจ ์‹คํ–‰ ์‹คํŒจ | ํ–‰๋ ฌ `exec-failed`(errno 2์™€ 13, pipe์™€ PTY) | +| ๊ด€์ฐฐ ์‹œ๊ฐ„ ์ดˆ๊ณผ | ์žฅ์•  `stalled-observer` | +| ์‚ด์•„๋‚จ์€ ์ž์† | ์žฅ์•  `descendant`, `escape` | +| ์ข…๋ฃŒ์™€ exit์˜ ๊ฒฝ์Ÿ | ์žฅ์•  `terminate-race` | +| shutdown๊ณผ exit์˜ ๊ฒฝ์Ÿ | ์žฅ์•  `shutdown` | +| runner ์—ฐ๊ฒฐ ๋Š๊น€ | ์ค€๋น„ ์ค‘์ธ ๊ฒฝ์šฐ๋Š” ์žฅ์•  `worker-killed`. launch ์ž์ฒด ์ค‘์— worker๋ฅผ ์žƒ๋Š” ๊ฒฝ์šฐ๋Š” ์Šคํฌ๋ฆฝํŠธ worker๋ฅผ ์“ฐ๋Š” gateway ์‹œํ—˜์ด ๋‹ค๋ฃธ | +| ์˜ค๋ž˜๋˜๊ฑฐ๋‚˜ ์ค‘๋ณต๋œ ์š”์ฒญ | ์žฅ์•  `authority-restart`(๋‹ค์‹œ ์žฌ๊ฐœํ•˜๋ฉด ๊ฐ™์€ ํ”„๋กœ์„ธ์Šค, ์‹คํ–‰ ํ•œ ๋ฒˆ)์™€ ์ œ์–ด ์ธก์ •์˜ ์ค‘๋ณต ์‹คํ–‰ ์ˆ˜. runner wire์—์„œ ์ค€๋น„ยทlaunch ๋ฉ”์‹œ์ง€๋ฅผ ๋‹ค์‹œ ๋ณด๋‚ด๋Š” ๊ฒฝ์šฐ๋Š” ์‹œํ—˜ํ•˜์ง€ ์•Š์œผ๋ฉฐ, ์•Œ๋ ค์ง„ ๊ณต๋ฐฑ์ž„ | +| ๋ถˆํ™•์‹คํ•œ ์‹คํ–‰ | ์žฅ์•  `authority-stalled`. ๋ถˆํ™•์‹คํ•œ attempt๊ฐ€ ๋‹ค์‹œ ํ—ˆ์šฉ๋˜๋ฉด ์ œ์–ด ์ธก์ •์ด ์‹คํŒจํ•จ | +| credential ์‹คํŒจ | ์žฅ์•  `credential` | +| journalยท์ €์žฅ์†Œ ๊ฑฐ๋ถ€ | ์ฃผ์ž…ํ•˜์ง€ ์•Š์Œ. fixture authority๋Š” journal์ด ๊ฑฐ๋ถ€ํ•˜๊ฒŒ ๋งŒ๋“ค ์ˆ˜ ์—†์Œ. ๊ฑฐ๋ถ€ ์ž์ฒด๋Š” DevGuard์˜ ์Šค์œ„ํŠธ๊ฐ€, ๊ทธ ์ฝ”๋“œ์˜ ๋Œ€์‘์€ CodeSpace ์‹œํ—˜์ด ๋‹ค๋ฃธ | + +fixture authority๋ฅผ ์“ฐ๋Š” gateway ์‹œํ—˜์€ DevGuard ์‹œํ—˜ ๋ฐ”์ด๋„ˆ๋ฆฌ๊ฐ€ ์žˆ๋Š” macOS์—์„œ๋งŒ ์‹คํ–‰๋˜๋ฉฐ, qualification ํ˜ธ์ŠคํŠธ์˜ [ํšŒ๊ท€ ๋‹จ๊ณ„](#regressions)์—์„œ ์‹คํ–‰ํ•ฉ๋‹ˆ๋‹ค. macOS CI๋Š” adapter ์‹œํ—˜, runner์˜ DevGuard ์‹œํ—˜, gateway์˜ `devguard` ๋ชจ๋“ˆ ์‹œํ—˜์„ ์‹คํ–‰ํ•˜์ง€๋งŒ ๊ทธ gateway ์‹œํ—˜์€ ์‹คํ–‰ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + + + +## ์ œ์–ด ๊ฒฝ๋กœ ๊ฐ€์šฉ์„ฑ + +๋ชฉํ‘œ๋Š” DevGuard๊ฐ€ ์ด๋ฏธ ์ •ํ•œ ๋ชฉํ‘œ ๊ฐ€์šด๋ฐ ์ œ์–ด ๊ฒฝ๋กœ์— ํ•ด๋‹นํ•˜๋Š” ๊ฒƒ์ด๋ฉฐ([SLO ์ ˆ์ฐจ](https://github.com/novelKR/DevGuard/blob/ea98d7fadf6ac10e9393db923105bf31f719952a/docs/planning/verification.md#slo-protocol)), ์—ฌ๊ธฐ์„œ ์ƒˆ๋กœ ์ •ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋กœ์ปฌ `process_status` p99 500 ms ์ดํ•˜, ์ข…๋ฃŒ ํ™•์ธ ์‘๋‹ต p99 1์ดˆ ์ดํ•˜(scope๊ฐ€ ์‹ค์ œ๋กœ ๋๋‚˜๋Š” ์‹œ๊ฐ„์€ ๋”ฐ๋กœ ๋ณด๊ณ ), MCP ์—ฐ๊ฒฐ ๋Š๊น€ ์—†์Œ, ๋‘ ๋ฒˆ ์‹คํ–‰๋œ ์‹คํ–‰ ์—†์Œ, ๋ถˆํ™•์‹คํ•œ ์‹คํ–‰์˜ ์žฌ์‹œ์ž‘ ์—†์Œ์ž…๋‹ˆ๋‹ค. ํฌ๊ทธ๋ผ์šด๋“œ ๋ธŒ๋ผ์šฐ์ € ๋ชฉํ‘œ๋Š” DG-1์˜ ๊ฒƒ์ด๋ฉฐ ์—ฌ๊ธฐ์„œ๋Š” ์ธก์ •ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ƒˆ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์˜ ๊ฑฐ๋ถ€ ์ง€์—ฐ๊ณผ ์ถœ๋ ฅ ์‘๋‹ต์„ฑ์€ ์ •ํ•ด์ง„ ๋ชฉํ‘œ๊ฐ€ ์—†์œผ๋ฏ€๋กœ ์ธก์ •ํ•ด ๋ณด๊ณ ํ•˜๋ฉฐ, ์„ค๊ณ„์˜ 250 ms ์ค€๋น„ ์˜ˆ์‚ฐ์„ ์ฐธ๊ณ ๊ฐ’์œผ๋กœ ์”๋‹ˆ๋‹ค. + + + +### ๊ตฌ๊ฐ„๊ณผ ํ‘œ๋ณธ + +`control`์€ runner ๋ชจ๋“œ(`in-process`, `uds`)๋งˆ๋‹ค ์„ธ ๋ฒˆ ๋ฐ˜๋ณตํ•ด ์ธก์ •ํ•ฉ๋‹ˆ๋‹ค. ํ•œ ๋ฒˆ์˜ ๋ฐ˜๋ณต์€ 10๋ถ„ ์œ ํœด ๊ตฌ๊ฐ„๊ณผ ๊ทธ ๋’ค 30๋ถ„ ์ด์ƒ์˜ ๋ถ€ํ•˜ ๊ตฌ๊ฐ„์ž…๋‹ˆ๋‹ค. ๊ตฌ๊ฐ„๋งˆ๋‹ค ์„ค์น˜๋œ ์„œ๋น„์Šค์— ๋“ฑ๋กํ•œ ์ƒˆ gateway๋ฅผ ์‹œ์ž‘ํ•˜๋ฉฐ, ๊ด€๋ฆฌ ๋Œ€์ƒ workspace๋Š” ๊ฐ๊ฐ 25 mCPU, 16 MiB, 2 tasks๋ฅผ ์š”์ฒญํ•ฉ๋‹ˆ๋‹ค. ํ‘œ๋ณธ์€ ๊ณ ์ • ์ผ์ •์œผ๋กœ ์ˆ˜์ง‘ํ•ฉ๋‹ˆ๋‹ค. + +| ์ผ์ • | ์ฃผ๊ธฐ | ์žฌ๋Š” ๊ฒƒ | +| --- | --- | --- | +| status | 1์ดˆ, ๋Œ€์ƒ ๋‘˜ | ์‹คํ–‰ ์ค‘์ธ ๊ด€๋ฆฌ ๋Œ€์ƒ `cat`(pipe)๊ณผ `sleep`(PTY)์˜ `process_status` | +| termination | 20์ดˆ | pipe์™€ PTY๋ฅผ ๋ฒˆ๊ฐˆ์•„ ์ƒˆ๋กœ ์‹œ์ž‘ํ•œ ๊ด€๋ฆฌ ๋Œ€์ƒ `sleep`. `terminate_process`์˜ ๋‹ต๊นŒ์ง€, ๊ทธ๋ฆฌ๊ณ  ๋”ฐ๋กœ exited ์ƒํƒœ๊นŒ์ง€, scope์˜ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน์ด ์‚ฌ๋ผ์งˆ ๋•Œ๊นŒ์ง€, DevGuard journal์— attempt ํ•ด์ œ๊ฐ€ ๋ณด์ผ ๋•Œ๊นŒ์ง€ | +| I/O | 5์ดˆ | `write_stdin`์œผ๋กœ `cat`์— ์“ด ๋ฒˆํ˜ธ ๋ถ™์€ ์ค„์ด `read_process`๋กœ ๋Œ์•„์˜ฌ ๋•Œ๊นŒ์ง€ | +| refusal | 30์ดˆ | work capacity๋ณด๋‹ค ๋งŽ์€ CPU๋ฅผ ์š”์ฒญํ•˜๋Š” workspace์˜ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰. ๊ฑฐ๋ถ€๋˜์–ด์•ผ ํ•จ | +| host | 5์ดˆ | ๋ฉ”๋ชจ๋ฆฌ pressure ์ˆ˜์ค€, swap, load average | +| service | 60์ดˆ | ์„œ๋น„์Šค๊ฐ€ ์—ฌ์ „ํžˆ ๊ทธ release๋ฅผ ์‹คํ–‰ํ•˜๋Š”์ง€์™€ ๊ทธ ํ”„๋กœ์„ธ์Šค ID | +| footprint | 5์ดˆ | ์‹คํ–‰ ์†Œ์œ ์ž์˜ RSS, thread, ํ”„๋กœ์„ธ์Šค, CPU(`uds` ๋ชจ๋“œ์—์„œ๋Š” gateway์˜ ๊ฐ’๋„). ์†Œ๋น„์ž์˜ control ์˜ˆ์•ฝ๊ณผ ๋น„๊ตํ•ด ๋ณด๊ณ  | + +์•ž ํ‘œ๋ณธ์ด ์•„์ง ๋๋‚˜์ง€ ์•Š์€ ๋™์•ˆ ์ง€๋‚˜๊ฐ„ ์ผ์ •์€ ๋†“์นœ ๊ฒƒ์œผ๋กœ ๊ธฐ๋กํ•˜๋ฉฐ, ๋Šฆ๊ฒŒ ์žฌ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ๋†“์นœ ์ผ์ •, ์˜ค๋ฅ˜, ์ด๋ฏธ ์‹คํ–‰ ์ค‘์ด ์•„๋‹Œ ๋Œ€์ƒ์— ๋Œ€ํ•œ ๋‹ต์€ ๋น ์ง„ ํ‘œ๋ณธ์ด๊ณ , ๋น ์ง„ ํ‘œ๋ณธ์€ DG-1์ฒ˜๋Ÿผ nearest-rank ๋ฐฑ๋ถ„์œ„์—์„œ ๋ชจ๋“  ๊ฐ’๋ณด๋‹ค ํฐ ์ˆœ์œ„๋กœ ์…‰๋‹ˆ๋‹ค. ์šฉ๋Ÿ‰์ด๋‚˜ pressure ๋•Œ๋ฌธ์— ํ—ˆ์šฉ์ด ๊ฑฐ๋ถ€๋œ(`RESOURCE_UNAVAILABLE`) ์ข…๋ฃŒ ๋Œ€์ƒ์€ ํ‘œ๋ณธ์ด ์•„๋‹™๋‹ˆ๋‹ค. ๊ตฌ๊ฐ„์ด ๋๋‚˜๋ฉด ๋‘ status ๋Œ€์ƒ์„ ์ข…๋ฃŒํ•˜๋ฉฐ, ๊ฐ ์ข…๋ฃŒ๋Š” ์‹ค์ œ๋กœ ํšจ๊ณผ๊ฐ€ ์žˆ์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. + + + +### ๋ถ€ํ•˜ + +๋ถ€ํ•˜๋Š” DG-1์˜ ๊ฒƒ์ž…๋‹ˆ๋‹ค. ์ฃผ์–ด์ง„ DevGuard checkout์˜ `scripts/measure.py`์— ์žˆ๋Š” ์†Œ๋น„์ž ์—ฌ์„ฏ์„, ์„ค์น˜๋œ release์˜ `devguard exec`๋กœ `devguard-qualify`์™€ release ์†Œ์Šค์˜ Cargo ๋นŒ๋“œ์™€ ํ•จ๊ป˜ ์‹คํ–‰ํ•ฉ๋‹ˆ๋‹ค. ๊ทธ๋ž˜์„œ pressure์™€ ๋Œ€๊ธฐ์—ด์€ ์„œ๋น„์Šค ์ž์‹ ์˜ ๊ฒƒ์ž…๋‹ˆ๋‹ค. `--load cold`(๊ธฐ๋ณธ)๋Š” ๋นˆ target ๋””๋ ‰ํ„ฐ๋ฆฌ์—์„œ ๋นŒ๋“œํ•˜๊ณ , `--load warm`์€ ๋จผ์ € target ๋””๋ ‰ํ„ฐ๋ฆฌ๋ฅผ ๋นŒ๋“œํ•˜๋ฉฐ ๊ทธ ๋นŒ๋“œ๊ฐ€ ์‹คํŒจํ•˜๋ฉด ํŒ์ • ๋ถˆ๊ฐ€์ž…๋‹ˆ๋‹ค. qualification ์‹คํ–‰์€ cold ์กฐ๊ฑด์„ ์”๋‹ˆ๋‹ค. warm ์‹คํ–‰์€ ์ถ”๊ฐ€ ์กฐ๊ฑด์ด๋ฉฐ, ํ•˜๋ฉด ๋”ฐ๋กœ ๋ณด๊ณ ํ•ฉ๋‹ˆ๋‹ค. ์†Œ๋น„์ž๋Š” ๊ตฌ๊ฐ„์˜ ์˜ˆ์ • ๋๊นŒ์ง€ ์‹คํ–‰ํ•˜๊ณ , ๊ทธ ๋’ค DG-1์˜ ์™„๋ฃŒ ํ•œ๋„ ์•ˆ์—์„œ ์ง„ํ–‰ ์ค‘์ธ ์‹คํ–‰์„ ๋งˆ์นฉ๋‹ˆ๋‹ค. ํ•˜๋„ค์Šค์˜ ๋Œ€์ƒ๊ณผ ๋ถ€ํ•˜๋Š” work capacity์˜ ์ ˆ๋ฐ˜, ๊ณง ๋ฉ”๋ชจ๋ฆฌ ๊ฒฝ๊ณ  ์•„๋ž˜์—์„œ DevGuard๊ฐ€ ์œ ์ง€ํ•˜๋Š” ์šฉ๋Ÿ‰ ์•ˆ์— ๋“ค์–ด๊ฐ‘๋‹ˆ๋‹ค. + + + +### ํŒ์ • + +๊ตฌ๊ฐ„์€ ๋‹ค์Œ๊ณผ ๊ฐ™์œผ๋ฉด **ํŒ์ • ๋ถˆ๊ฐ€**์ž…๋‹ˆ๋‹ค. ์ด๋Š” ์ž˜๋ชป๋œ ๊ด€์ฐฐ์ด๋ฉฐ ํ†ต๊ณผ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. ์‹œ์ž‘ํ•  ๋•Œ ๋‹ค๋ฅธ ์ž‘์—…์ด ์ž์›์„ ์ฐจ์ง€ํ•˜๊ณ  ์žˆ์—ˆ๊ฑฐ๋‚˜, journal์„ ์ฝ์„ ์ˆ˜ ์—†์—ˆ๊ฑฐ๋‚˜, status ๋Œ€์ƒ์„ ์‹œ์ž‘ํ•˜์ง€ ๋ชปํ–ˆ๊ฑฐ๋‚˜, ์›์‹œ ์ฆ๊ฑฐ๋ฅผ ์“ฐ์ง€ ๋ชปํ–ˆ๊ฑฐ๋‚˜, ์„œ๋น„์Šค๋ฅผ ๊ด€์ฐฐํ•  ์ˆ˜ ์—†์—ˆ๊ฑฐ๋‚˜ ๋‹ค๋ฅธ release๊ฐ€ ์„ ํƒ๋˜์—ˆ๊ฑฐ๋‚˜, ๋ถ€ํ•˜๊ฐ€ ํ•œ๋„ ์•ˆ์— ๋๋‚˜์ง€ ์•Š์€ ๊ฒฝ์šฐ์ž…๋‹ˆ๋‹ค. + +๊ทธ๋ ‡์ง€ ์•Š์œผ๋ฉด ๋‹ค์Œ ์ค‘ ํ•˜๋‚˜์ผ ๋•Œ **์‹คํŒจ**์ž…๋‹ˆ๋‹ค. status p99๊ฐ€ 500 ms๋ฅผ ๋„˜์Œ, ํ™•์ธ ์‘๋‹ต p99๊ฐ€ 1์ดˆ๋ฅผ ๋„˜์Œ, ์ข…๋ฃŒ๊ฐ€ ํ™•์ธ๋˜์—ˆ๋Š”๋ฐ ๋Œ€์ƒ์ด ๋๋‚˜์ง€ ์•Š์Œ, ์ข…๋ฃŒํ•œ ๋Œ€์ƒ์ด ํ•ด์ œ๋˜์ง€ ์•Š์Œ, I/O ์ค„์ด ๋Œ์•„์˜ค์ง€ ์•Š์Œ, ์–ด๋–ค ํ‘œ๋ณธ์—์„œ๋“  MCP ์—ฐ๊ฒฐ์ด ๋Š๊น€, status ๋Œ€์ƒ์˜ ๋งˆ์ง€๋ง‰ ์ข…๋ฃŒ๊ฐ€ ํšจ๊ณผ๊ฐ€ ์—†์Œ, ์ธก์ • ์ค‘์ธ release๊ฐ€ ์žฌ์‹œ์ž‘ํ–ˆ๊ฑฐ๋‚˜ ๋น„์ •์ƒ์ด ๋จ, ๋ถ€ํ•˜ ์‹คํ–‰์ด ๋ถˆํ™•์‹คํ•˜๊ฑฐ๋‚˜ ๋ถ€ํ•˜ attempt๊ฐ€ ์ž์›์„ ๊ณ„์† ์ฐจ์ง€ํ•จ, ํ•˜๋„ค์Šค์˜ attempt๊ฐ€ ์ž์›์„ ๊ณ„์† ์ฐจ์ง€ํ•จ, ํ•œ ์‹คํ–‰์ด ๋‘ ๋ฒˆ ์ด์ƒ ์‹คํ–‰๋จ, ๋ถˆํ™•์‹คํ•œ attempt๊ฐ€ ๋‹ค์‹œ ํ—ˆ์šฉ๋จ. + +๊ด€์ฐฐ์ด ๋„ˆ๋ฌด ์ ์–ด๋„ **ํŒ์ • ๋ถˆ๊ฐ€**์ž…๋‹ˆ๋‹ค. status ํ‘œ๋ณธ์ด ์ผ์ •์˜ 90%๋ณด๋‹ค ์ ๊ฑฐ๋‚˜, ์ข…๋ฃŒ ํ‘œ๋ณธ์ด 80%๋ณด๋‹ค ์ ๊ฑฐ๋‚˜, ๋ถ€ํ•˜๊ฐ€ DG-1์˜ ๊ฒƒ์ด ์•„๋‹Œ ๊ฒฝ์šฐ(์„ฑ๊ณตํ•œ ์‹คํ–‰์ด ์—†๋Š” ์†Œ๋น„์ž, ๊ตฌ๊ฐ„์˜ 4๋ถ„์˜ 1๋ณด๋‹ค ์งง๊ฒŒ ์‹คํ–‰๋œ ์†Œ๋น„์ž, ์‹คํŒจํ•œ ์‹คํ–‰์ด ์žˆ๋Š” ์†Œ๋น„์ž, ํ—ˆ์šฉ๋œ ๋ถ€ํ•˜๊ฐ€ ๊ตฌ๊ฐ„์˜ ์ ˆ๋ฐ˜๋ณด๋‹ค ์งง๊ฒŒ ์‹คํ–‰๋จ)์ž…๋‹ˆ๋‹ค. ๊ทธ ๋ฐ–์—๋Š” **ํ†ต๊ณผ**์ž…๋‹ˆ๋‹ค. + +๋ฐ˜๋ณต์€ ๋‘ ๊ตฌ๊ฐ„์ด ๋ชจ๋‘ ํ†ต๊ณผํ•˜๊ณ  ๊ทธ ๋’ค์—๋„ ์„œ๋น„์Šค๊ฐ€ ๊ฐ™์€ ํ”„๋กœ์„ธ์Šค๋กœ ๊ทธ release๋ฅผ ์‹คํ–‰ํ•  ๋•Œ ํ†ต๊ณผ์ž…๋‹ˆ๋‹ค. ์œ ํœด ๊ตฌ๊ฐ„์ด ์‹คํŒจํ•˜๋ฉด ๊ทธ ๋ฐ˜๋ณต์€ ํŒ์ • ๋ถˆ๊ฐ€์ด๋ฉฐ, ๋ถ€ํ•˜์˜ ์‹คํŒจ๋กœ ๋ณด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. runner ๋ชจ๋“œ๋Š” ์„ธ ๋ฐ˜๋ณต์ด ๋ชจ๋‘ ํ†ต๊ณผํ•˜๋ฉด qualification์„ ํ†ต๊ณผํ•˜๊ณ , ํ•˜๋‚˜๋ผ๋„ ์‹คํŒจํ•˜๋ฉด ์‹คํŒจํ•ฉ๋‹ˆ๋‹ค. ๋ฐ˜๋ณต ์‚ฌ์ด์— ๊ฐ’์„ ํ•ฉ์น˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์‹คํ–‰ ์ „์ฒด๋Š” ์ „์ฒด ์ ˆ์ฐจ(๋‘ ๋ชจ๋“œ, ์„ธ ๋ฐ˜๋ณต, 10๋ถ„๊ณผ 30๋ถ„)์—์„œ ๋‘ ๋ชจ๋“œ๊ฐ€ ๋ชจ๋‘ ํ†ต๊ณผํ•  ๋•Œ๋งŒ qualification์„ ํ†ต๊ณผํ•˜๊ณ , ์–ด๋Š ํ•œ์ชฝ์ด ์‹คํŒจํ•˜๋ฉด ์‹คํŒจํ•ฉ๋‹ˆ๋‹ค. `--rehearsal`(์œ ํœด 30์ดˆ, ๋ถ€ํ•˜ 60์ดˆ, ํ•œ ๋ฒˆ ๋ฐ˜๋ณต)์€ ์‹คํ–‰์„ ์‹œํ—˜ํ•ด ๋ณด๋Š” ๊ฒƒ์ด๋ฉฐ qualification์„ ํ†ต๊ณผํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + + + +### authority ์žฅ์•  + +`authority`๋Š” fixture authority๊ฐ€ ์ •์ƒ์ด ์•„๋‹ ๋•Œ์˜ ํ”„๋กœ์„ธ์Šค ์ œ์–ด๋ฅผ, runner ๋ชจ๋“œ๋งˆ๋‹ค 15์ดˆ์งœ๋ฆฌ ๋‹ค์„ฏ ๋‹จ๊ณ„๋กœ, ๊ณ„์† ์‹คํ–‰ ์ค‘์ธ ๊ฐ™์€ status ๋Œ€์ƒ์— ๋Œ€ํ•ด ์ธก์ •ํ•ฉ๋‹ˆ๋‹ค. ๋‹จ๊ณ„๋Š” `baseline`, `paused`(reconciler ๋ฉˆ์ถค), `stalled`(๋‹จ๊ณ„ ๋‚ด๋‚ด lock์„ ์žก์Œ), `restarted`, `stopped`(ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฃฝ์ž„)์ž…๋‹ˆ๋‹ค. status๋Š” 1์ดˆ๋งˆ๋‹ค, I/O echo๋Š” 3์ดˆ๋งˆ๋‹ค ์žฌ๊ณ , ์žฅ์•  ์ „์— ์‹œ์ž‘ํ•œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์…‹์„ 3, 7, 11์ดˆ์— ์ข…๋ฃŒํ•˜๋ฉฐ, 1, 5, 9, 13์ดˆ์— ์ƒˆ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์„ ์‹œ๋„ํ•ฉ๋‹ˆ๋‹ค. ๋‹จ๊ณ„๋งˆ๋‹ค ์œ„์˜ ๊ตฌ๊ฐ„ ๊ทœ์น™์œผ๋กœ ํŒ์ •ํ•˜๋ฉฐ, ์ƒˆ ์‹คํ–‰์€ authority๊ฐ€ ๋‹ตํ•  ์ˆ˜ ์žˆ๋Š” ๋™์•ˆ(`baseline`, `paused`, `restarted`)์—๋Š” ํ—ˆ์šฉ๋˜๊ณ , ๋‹ตํ•  ์ˆ˜ ์—†๋Š” ๋™์•ˆ(`stalled`, `stopped`)์—๋Š” `RESOURCE_AUTHORITY_UNAVAILABLE`์ด๋‚˜ `ADMISSION_UNKNOWN`์œผ๋กœ ๊ฑฐ๋ถ€๋˜์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ์žฌ์‹œ์ž‘ํ•œ ๋’ค๋‚˜ ๋ฉˆ์ถ˜ ๋’ค์—๋Š” DevGuard๊ฐ€ ์žฅ์•  ์ „์— ์‹œ์ž‘ํ•œ ๋Œ€์ƒ์„ ํ•ด์ œํ•  ์ˆ˜ ์—†์œผ๋ฏ€๋กœ, ๊ทธ ๋‹จ๊ณ„์—์„œ ํ•ด์ œ๋˜์ง€ ์•Š์€ ๋Œ€์ƒ์€ ์‹คํŒจ๊ฐ€ ์•„๋‹ˆ๋ผ ๋ณด๊ณ  ๋Œ€์ƒ์ž…๋‹ˆ๋‹ค. + + + +## ์ž์› ํšŒ๊ณ„ + +ํ•˜๋„ค์Šค๋Š” CodeSpace ์ž์‹ ์˜ ํŒ๋‹จ์ด ์•„๋‹ˆ๋ผ DevGuard์˜ ๊ธฐ๋ก์„ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. ํ–‰๋ ฌ์˜ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰๋งˆ๋‹ค gateway๊ฐ€ ๋กœ๊ทธ์— ๋‚จ๊ธด attempt๋ฅผ journal์—์„œ ์ฝ์–ด, ์˜ˆ์•ฝํ•˜๊ณ  ์ ์šฉํ•œ ์–‘์ด workspace์˜ ์š”์ฒญ๊ณผ ๊ฐ™์€์ง€, CPUยท๋ฉ”๋ชจ๋ฆฌยทtasks๊ฐ€ ๋ชจ๋‘ `applied`์ธ์ง€, scope root๊ฐ€ gateway๊ฐ€ ์‹คํ–‰ํ•œ ํ”„๋กœ์„ธ์Šค์ธ์ง€, scope๊ฐ€ ๋๋‚˜๋ฉด attempt๊ฐ€ `scope_terminated`๋กœ ํ•ด์ œ๋˜์–ด ์ž์›์„ ์ฐจ์ง€ํ•˜์ง€ ์•Š๊ณ  ์ถ”์ ์ด ๋๊นŒ์ง€ ์œ ์ง€๋˜์—ˆ๋Š”์ง€ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. ๊ฑฐ๋ถ€๋œ ํ˜ธ์ถœ์€ ๊ทธ attempt๋ฅผ ๋ฐํžˆ๊ณ , DevGuard๋Š” ๊ทธ attempt๋ฅผ ๊ธฐ๋Œ€ํ•œ ๋‹จ๊ณ„(`denied`, `cancelled`, `released`)๋กœ ์ž์› ์—†์ด ๊ธฐ๋กํ•˜๊ฑฐ๋‚˜, ๊ฑฐ๋ถ€๊ฐ€ ๋จผ์ € ์ผ์–ด๋‚œ ๊ฒฝ์šฐ์—๋Š” attempt๊ฐ€ ์—†์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ์ œ์–ด ๋ถ€ํ•˜ ์•„๋ž˜์—์„œ ์žฐ gateway์™€ worker์˜ footprint๋Š” `codespace` ์˜ˆ์•ฝ๊ณผ ๋น„๊ตํ•ด ๋ณด๊ณ ํ•ฉ๋‹ˆ๋‹ค. ์˜ˆ์•ฝ์€ DevGuard์˜ ํšŒ๊ณ„ ์ถ”์ •์ด๋ฏ€๋กœ ์ด ๋น„๊ต๋Š” ํŒ์ •์— ์“ฐ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + + + +## ํšŒ๊ท€์™€ ๋™๋“ฑ์„ฑ + +ํ›„๋ณด ์กฐํ•ฉ์—์„œ, qualification ํ˜ธ์ŠคํŠธ์˜ ์ตœ์†Œ ํ™˜๊ฒฝ์œผ๋กœ `python3 scripts/validate-upstream.py all macos-core macos-fork-race`๋ฅผ ์‹คํ–‰ํ•ด ๋ชจ๋“  ๋‹จ๊ณ„๋ฅผ ๋Œ๋ฆฝ๋‹ˆ๋‹ค. DevGuard ์‹œํ—˜ ๋ฐ”์ด๋„ˆ๋ฆฌ๋ฅผ ์“ฐ๋Š” ๋‹จ๊ณ„์—์„œ๋Š” ๊ทธ ๋ฐ”์ด๋„ˆ๋ฆฌ๊ฐ€ ๊ผญ ์žˆ์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. Linux๋Š” ํ›„๋ณด head์˜ CI๊ฐ€ ๋ณด์—ฌ ์ฃผ๋ฉฐ, ๊ทธ `dependencies` ๋‹จ๊ณ„๊ฐ€ `devguard` feature๋ฅผ ์ผ  ์˜์กด์„ฑ ๊ทธ๋ž˜ํ”„์™€ ๋ˆ ๊ทธ๋ž˜ํ”„๋ฅผ ๋ชจ๋‘ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. ์–ด๋Š ๋‹จ๊ณ„์—์„œ๋“  ํšŒ๊ท€๊ฐ€ ์žˆ์œผ๋ฉด qualification์€ ์‹คํŒจํ•ฉ๋‹ˆ๋‹ค. + + + +## ์ฆ๊ฑฐ + +`verify`๋ฅผ ๋บ€ ๋ชจ๋“  ์Šค์œ„ํŠธ๋Š” ์ƒˆ ๋””๋ ‰ํ„ฐ๋ฆฌ๋ฅผ ๋งŒ๋“ค์–ด ์“ฐ๊ณ  ๋ด‰์ธํ•ฉ๋‹ˆ๋‹ค. ์—ฌ๊ธฐ์—๋Š” `report.json`(์Šค์œ„ํŠธ, ์ธ์ž, ํ•˜๋„ค์Šค ํŒŒ์ผ hash์™€ commit, ํŒ์ •, ๋งˆ์ง€๋ง‰์— ์ •๋ฆฌํ•œ ๋‚ด์šฉ), ์Šค์œ„ํŠธ์˜ ๊ธฐ๋ก, ์›์‹œ ํ‘œ๋ณธ์˜ JSON lines๊ฐ€ ๋“ค์–ด๊ฐ€๋ฉฐ, ๊ฐ gateway์˜ MCP ๋Œ€ํ™” ๊ธฐ๋ก๊ณผ ๋กœ๊ทธ๋„ ํฌํ•จ๋ฉ๋‹ˆ๋‹ค. ๋ด‰์ธ์€ ๋ชจ๋“  ํŒŒ์ผ์˜ SHA-256์„ `MANIFEST.json`์— ๊ธฐ๋กํ•˜๊ณ  ํŒŒ์ผ์„ ์ฝ๊ธฐ ์ „์šฉ์œผ๋กœ ๋ฐ”๊ฟ‰๋‹ˆ๋‹ค. ์‹คํŒจ, ํŒ์ • ๋ถˆ๊ฐ€, ์ค‘๋‹จ๋œ ์‹คํ–‰๋„ ๋ด‰์ธํ•ด ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. ์–ด๋–ค ๋””๋ ‰ํ„ฐ๋ฆฌ์—๋„ credential, credential digest, ์ž์‹ ํ”„๋กœ์„ธ์Šค์˜ ํ™˜๊ฒฝ์€ ๋“ค์–ด๊ฐ€์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + +`qualify-devguard.py verify <๋””๋ ‰ํ„ฐ๋ฆฌ>`๋Š” manifest๋ฅผ ํ™•์ธํ•˜๊ณ  ์›์‹œ ํŒŒ์ผ์—์„œ ๋ชจ๋“  ํŒ์ •์„ ๋‹ค์‹œ ๊ณ„์‚ฐํ•ฉ๋‹ˆ๋‹ค. ์‹œ๋‚˜๋ฆฌ์˜ค ํŒ์ •์€ ๊ทธ ํ™•์ธ ํ•ญ๋ชฉ์—์„œ, authority ๋‹จ๊ณ„์™€ ์ œ์–ด ๊ตฌ๊ฐ„์€ ๊ฐ์ž์˜ ํ‘œ๋ณธ์—์„œ, ๊ทธ๋‹ค์Œ ๋ฐ˜๋ณต, runner ๋ชจ๋“œ, ์‹คํ–‰ ์ „์ฒด ์ˆœ์„œ๋กœ ๊ณ„์‚ฐํ•ฉ๋‹ˆ๋‹ค. ์‹คํ–‰ํ–ˆ๋˜ ํ•˜๋„ค์Šค์™€ ์ง€๊ธˆ ํ™•์ธํ•˜๋Š” ํ•˜๋„ค์Šค๊ฐ€ ๊ฐ™์€์ง€๋„ ๋ณด๊ณ ํ•˜๋ฉฐ, ์•„๋ฌด๊ฒƒ๋„ ์“ฐ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. + + + +## ๊ธฐ์กด `off` ๊ฒฝ๋กœ ๊ฒฐ์ • + +์ž‘์—… ์ง€์‹œ์„œ๊ฐ€ ๊ธฐ์กด backend์— ๋Œ€ํ•ด ์š”๊ตฌํ•˜๋Š” ๊ฒฐ์ • ๊ธฐ๋ก(CSRG-C09)์ž…๋‹ˆ๋‹ค. + +**๊ฒฐ์ •:** ์ฐธ์—ฌ `off`๋ฅผ ํ˜ธํ™˜ ๊ฒฝ๋กœ์ด์ž ๊ธฐ๋ณธ๊ฐ’์œผ๋กœ, ๋ฐ”๊พธ์ง€ ์•Š๊ณ  ์œ ์ง€ํ•ฉ๋‹ˆ๋‹ค. + +- **๋ฒ”์œ„.** ๊ธฐ๋ณธ๊ฐ’์ด์ž ์„ค์ •์ด ์—†์„ ๋•Œ์˜ ์˜๋ฏธ์ธ ์ฐธ์—ฌ `off`. ๋ชจ๋“  ํ”Œ๋žซํผ๊ณผ runner ๋ชจ๋“œ, pipe์™€ PTY์— ํ•ด๋‹นํ•ฉ๋‹ˆ๋‹ค. ์ด ์‹คํ–‰์€ ์š”์ฒญํ•œ ๋ช…๋ น ์ž์ฒด์ž…๋‹ˆ๋‹ค. pipe ์ž์‹์€ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰๊ณผ ๊ฐ™์€ CodeSpace์˜ reaper ํ•˜๋‚˜๋ฅผ ์“ฐ๊ณ , PTY ์ž์‹์€ reap ์ „ ๊ด€์ฐฐ ์ฐฝ์ด ์—†๋Š” ๊ณ ์ • Codex PTY spawn์„ ์”๋‹ˆ๋‹ค. +- **์ด์œ .** ์ž‘์—… ์ง€์‹œ์„œ๋Š” ๊น”๋”ํ•จ๋งŒ์„ ์œ„ํ•ด ์ฝ”๋“œ๋ฅผ ์˜ฎ๊ธฐ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. `off`๋Š” ์ œํ’ˆ์˜ ๊ธฐ๋ณธ๊ฐ’์ด๊ณ , `required`๋Š” ์šด์˜์ž๊ฐ€ ๋ช…์‹œ์ ์œผ๋กœ ๊ณ ๋ฅด๋Š” ๊ฒƒ์ด๋ฉฐ DG-LINUX ์ „๊นŒ์ง€ Linux์—์„œ๋Š” ์ง€์›ํ•˜์ง€ ์•Š์œผ๋ฏ€๋กœ, Linux์—์„œ๋Š” `off`๊ฐ€ ์œ ์ผํ•œ ๊ฒฝ๋กœ์ž…๋‹ˆ๋‹ค. ๊ด€๋ฆฌํ˜• ์‹คํ–‰๋งŒ ๋‚จ๊ธฐ๋ฉด Linux์˜ ๋ชจ๋“  ์‹คํ–‰๊ณผ DevGuard๊ฐ€ ์—†๋Š” ๋ชจ๋“  ํ™˜๊ฒฝ์ด ์‚ฌ๋ผ์ง‘๋‹ˆ๋‹ค. +- **์ค‘๋ณต๋œ ๋ถ€๋ถ„.** spawn ๊ฒฝ๋กœ๋Š” ํ•˜๋‚˜์ด๋ฉฐ ์‹คํ–‰ํ•  ํ”„๋กœ๊ทธ๋žจ๋งŒ ๊ณ ๋ฆ…๋‹ˆ๋‹ค. ์š”์ฒญํ•œ ๋ช…๋ น์ด๊ฑฐ๋‚˜, permit๊ณผ report descriptor๋ฅผ ๋ฐ›๋Š” DevGuard์˜ launch helper์ž…๋‹ˆ๋‹ค([`crates/runner/src/process.rs`](../../crates/runner/src/process.rs)์˜ `How::Direct`์™€ `How::Managed`). reap์€ ์ค‘๋ณต๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์–ด๋Š ์ชฝ์ด๋“  pipe ์ž์‹์€ CodeSpace์˜ reaper ํ•˜๋‚˜๊ฐ€, PTY ์ž์‹์€ ๊ณ ์ • Codex PTY์˜ waiter๊ฐ€ reapํ•ฉ๋‹ˆ๋‹ค. PTY๋Š” ํ˜ธ์ถœ ํ•˜๋‚˜๋งŒ ๋‹ค๋ฆ…๋‹ˆ๋‹ค. ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” ์‹คํ–‰์€ `spawn_inheriting`, ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ [๊ธฐ๋ก๋œ ํŒจ์น˜](upstream-patches.md)์˜ `spawn_observed`๋ฅผ ์”๋‹ˆ๋‹ค. ๊ด€๋ฆฌ ๋Œ€์ƒ ๊ฒฝ๋กœ์—๋Š” ์‹คํ–‰ ํ—ˆ์šฉ, launch, ์ •๋ฆฌ๊ฐ€ ๋”ํ•ด์ง€๋ฉฐ([`crates/runner/src/launch.rs`](../../crates/runner/src/launch.rs)), `off`๋Š” ์ด ๊ฒฝ๋กœ์— ๋“ค์–ด๊ฐ€์ง€ ์•Š์Šต๋‹ˆ๋‹ค. `devguard` feature๊ฐ€ ์—†์œผ๋ฉด ๋นŒ๋“œ์— DevGuard ํŒจํ‚ค์ง€๊ฐ€ ์—†๊ณ , ์žˆ์œผ๋ฉด DevGuard์˜ client์™€ contract crate๋งŒ ๋“ค์–ด๊ฐ€๋ฉฐ, `dependencies` ๋‹จ๊ณ„๊ฐ€ ์ด๋ฅผ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. +- **`off`๊ฐ€ ์ œ๊ณตํ•˜์ง€ ์•Š๋Š” ๊ฒƒ.** DevGuard์˜ ์‹คํ–‰ ํ—ˆ์šฉ, ์˜ˆ์•ฝ, ํšŒ๊ณ„๊ฐ€ ์—†๊ณ , scope ๋‹จ์†์ด ์—†์œผ๋ฉฐ, ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” PTY root๋ฅผ reap ์ „์— ๊ด€์ฐฐํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. Linux ๋ช…๋ น sandbox๋Š” `off` ์‹คํ–‰๋งŒ ์“ธ ์ˆ˜ ์žˆ๊ณ , sandbox๊ฐ€ ์ผœ์ง„ ๊ณณ์—์„œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค. +- **์žฌ๊ฒ€ํ†  ์‹œ์ .** DG-LINUX๊ฐ€ Linux ๊ฐ•์ œ ๋ณดํ˜ธ๋ฅผ qualificationํ•  ๋•Œ, Codex pin ์ด๋™์œผ๋กœ PTY spawn์ด ๋ฐ”๋€” ๋•Œ, `required`๊ฐ€ ๊ธฐ๋ณธ๊ฐ’์ด ๋  ๋•Œ์ž…๋‹ˆ๋‹ค. +- **์ œ๊ฑฐ ๊ธฐ์ค€.** ๋…๋ฆฝ์ ์ธ ์ œํ’ˆ์ƒ์˜ ์ด์œ ๊ฐ€ ์žˆ๊ณ , ์ด ๊ฒฝ๋กœ ์—†์ด๋„ ์ง€์›ํ•˜๋Š” ๋ชจ๋“  ๋ชจ๋“œ์˜ ๋™์ž‘์ด ์œ ์ง€๋œ๋‹ค๋Š” ํšŒ๊ท€ ์ฆ๊ฑฐ๊ฐ€ ์žˆ์„ ๋•Œ๋งŒ ์ œ๊ฑฐํ•ฉ๋‹ˆ๋‹ค. diff --git a/docs/ko/devguard-integration.md b/docs/ko/devguard-integration.md index f2ae02d..7bbaec8 100644 --- a/docs/ko/devguard-integration.md +++ b/docs/ko/devguard-integration.md @@ -8,7 +8,7 @@ [DevGuard](https://github.com/novelKR/DevGuard)๋Š” ๊ฐœ๋ฐœ ์ž‘์—…์˜ ์ž์›์„ ์ค‘์•™์—์„œ ๊ด€๋ฆฌํ•˜๋Š” ๋…๋ฆฝ ์‹œ์Šคํ…œ์ด๋ฉฐ CodeSpace์™€ ๋™์ผํ•œ Apache-2.0 ๋ผ์ด์„ ์Šค๋ฅผ ์ ์šฉํ•œ๋‹ค. ์Šน์ธ๋œ ๊ฒฐํ•ฉ ๊ฒฝ๋กœ์— ๋”ฐ๋ผ ์‹คํ–‰ ํ—ˆ์šฉ๊ณผ ์ž์› ํšŒ๊ณ„๋ฅผ ๊ณตํ†ต ๊ณ„์ธต์— ๋งก๊ธฐ๊ณ , CodeSpace๋Š” ํ”„๋กœ์„ธ์Šค ์†Œ์œ ๊ถŒ, PTY, ์ž…์ถœ๋ ฅ, ๊ถŒํ•œ, ์Šน์ธ hold์™€ workspace ์กฐ์ •์„ ๊ณ„์† ๋‹ด๋‹นํ•œ๋‹ค. -**ํ˜„์žฌ ์ƒํƒœ:** DevGuard๋Š” ๋…๋ฆฝ ์ €์žฅ์†Œ์—์„œ DG-1์„ ์™„๋ฃŒํ–ˆ๋‹ค([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [๋งˆ์ผ์Šคํ†ค ์›์žฅ](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). DG-0 ๊ณ„์•ฝ ์œ„์— macOS ์‹ค์ œ ํ˜ธ์ŠคํŠธ ์ฆ๊ฑฐ, ํ˜„์žฌ ์‚ฌ์šฉ์ž์˜ LaunchAgent๋กœ ์„ค์น˜ํ•˜๋Š” ์ธ์ฆ daemon, fenced launch helper์™€ ๋Œ€์กฐ, Cargo adapter๋ฅผ ๊ฐ–์ถ˜ `devguard` ๋ช…๋ นํ–‰ owner, ํ›„๋ณด ์‹œํ—˜์šฉ parent lease, upgrade์™€ repair๋ฅผ ์ œ๊ณตํ•œ๋‹ค. ์ธก์ •ํ•œ ํ˜ธ์ŠคํŠธ์™€ ์ •์ฑ…์—์„œ release `0.1.0-5daee5d-b3fa569e`(Git tag๊ฐ€ ์•„๋‹Œ release ID)์˜ macOS SLO qualification์„ ๋งˆ์ณค์œผ๋ฉฐ Linux ๊ฐ•์ œ ๋ณดํ˜ธ๋Š” qualificationํ•˜์ง€ ์•Š์•˜๋‹ค. CodeSpace ์†Œ๋น„(CS-RG)๋Š” ์ง„ํ–‰ ์ค‘์ด๋ฉฐ qualification์„ ๋งˆ์น˜์ง€ ์•Š์•˜๋‹ค. CSRG-U1์€ opt-in ์ƒํƒœ ์ „์šฉ ์—ฐ๊ฒฐ์„ ์ถ”๊ฐ€ํ–ˆ๋‹ค. `devguard` feature๋กœ ๋นŒ๋“œํ•˜๊ณ  `--devguard status`๋กœ ์‹œ์ž‘ํ•œ gateway๋Š” [`6e7e065`](upstream-lock.md#devguard-client-pin)์— ๊ณ ์ •ํ•œ DevGuard client๋กœ DevGuard์˜ ์ƒํƒœ๋ฅผ `workspace_info`์— ๋ณด๊ณ ํ•œ๋‹ค. CSRG-U2๋ถ€ํ„ฐ๋Š” CodeSpace์˜ ์‹คํ–‰์„ ์†Œ์œ ํ•œ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์Šค์Šค๋กœ๋ฅผ DevGuard์— ๋“ฑ๋กํ•˜๋ฉฐ(`--devguard register`), workspace์˜ `resources` ์„ค์ •์œผ๋กœ ์ž์› ์ฐธ์—ฌ๋ฅผ ํ•„์ˆ˜๋กœ ์ง€์ •ํ•  ์ˆ˜ ์žˆ๋‹ค. CSRG-U3๋ถ€ํ„ฐ ๊ทธ ์†Œ์œ ์ž๋Š” ์ด๋Ÿฐ ์‹คํ–‰๋งˆ๋‹ค ๋ฌด์—‡์„ ๋งŒ๋“ค๊ธฐ ์ „์— ํ”„๋กœ์„ธ์Šค slot์„ ์žก๊ณ  DevGuard์˜ ์‹คํ–‰ ํ—ˆ์šฉ์„ ๋ฐ›๋Š”๋‹ค. CSRG-U4๋ถ€ํ„ฐ๋Š” launch helper๋ฅผ ์„ค์ •ํ–ˆ์œผ๋ฉด(`--devguard-launch-helper`) ๊ทธ ์‹คํ–‰์„ DevGuard์˜ launch helper๋กœ ํ•œ ๋ฒˆ ์‹คํ–‰ํ•œ๋‹ค. DevGuard๋Š” helper์˜ scope๋ฅผ ๋ฌถ๊ณ  ์ •์ฑ…์„ ์ ์šฉํ•œ ๋’ค์— ์‹คํ–‰์„ ์Šน์ธํ•œ๋‹ค. helper๊ฐ€ ์—†์œผ๋ฉด ์‹œ์ž‘ํ•˜์ง€ ์•Š์€ attempt๋ฅผ ์ทจ์†Œํ•˜๊ณ  ์‹คํ–‰์„ ๊ฑฐ๋ถ€ํ•œ๋‹ค. DevGuard ์—†์ด ์‹คํ–‰ํ•˜๋Š” ๊ฒƒ์€ ์—†๋‹ค. CSRG-U5๋ถ€ํ„ฐ ํ”„๋กœ์„ธ์Šค๋งˆ๋‹ค reaper๋Š” ํ•˜๋‚˜๋‹ค. DevGuard๋Š” ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์˜ scope๋ฅผ root๊ฐ€ reap๋˜๊ธฐ ์ „์— ๊ด€์ฐฐํ•œ๋‹ค. pipe์—์„œ๋Š” CodeSpace ์ž์‹ ์˜ reaper๊ฐ€, PTY์—์„œ๋Š” [๊ธฐ๋ก๋œ ํŒจ์น˜](upstream-lock.md#codex-downstream-patch)๋ฅผ ์ ์šฉํ•œ ๊ณ ์ • Codex PTY์˜ waiter๊ฐ€ ๊ทธ ๊ธฐํšŒ๋ฅผ ์ค€๋‹ค. ๊ด€๋ฆฌ ๋Œ€์ƒ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฃฝ์ด๋ฉด ๊ทธ scope๋„ ์ข…๋ฃŒํ•˜๋ฉฐ, ์‹œ์ž‘ํ•˜์ง€ ์•Š์•˜์Œ์ด ์ฆ๋ช…๋œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์˜ ์Šน์ธ์€ ๋‹ค์‹œ ์žฌ๊ฐœํ•  ์ˆ˜ ์žˆ๋‹ค. qualification์€ ์ดํ›„ ๋‹จ์œ„(CSRG-U6)๋‹ค. qualification์„ ๋งˆ์นœ release๋Š” ์‹คํ–‰์— ์“ธ ํ›„๋ณด๋กœ ๋‚จ์œผ๋ฉฐ ์„ ํƒ๋œ pin์ด ์•„๋‹ˆ๋‹ค. client pin์€ client crate์˜ ์†Œ์Šค pin์ด๋‹ค. +**ํ˜„์žฌ ์ƒํƒœ:** DevGuard๋Š” ๋…๋ฆฝ ์ €์žฅ์†Œ์—์„œ DG-1์„ ์™„๋ฃŒํ–ˆ๋‹ค([`395315d`](https://github.com/novelKR/DevGuard/tree/395315d34b5d458ea1774446727f0cb14bd8a120), [๋งˆ์ผ์Šคํ†ค ์›์žฅ](https://github.com/novelKR/DevGuard/blob/395315d34b5d458ea1774446727f0cb14bd8a120/milestones.json)). DG-0 ๊ณ„์•ฝ ์œ„์— macOS ์‹ค์ œ ํ˜ธ์ŠคํŠธ ์ฆ๊ฑฐ, ํ˜„์žฌ ์‚ฌ์šฉ์ž์˜ LaunchAgent๋กœ ์„ค์น˜ํ•˜๋Š” ์ธ์ฆ daemon, fenced launch helper์™€ ๋Œ€์กฐ, Cargo adapter๋ฅผ ๊ฐ–์ถ˜ `devguard` ๋ช…๋ นํ–‰ owner, ํ›„๋ณด ์‹œํ—˜์šฉ parent lease, upgrade์™€ repair๋ฅผ ์ œ๊ณตํ•œ๋‹ค. ์ธก์ •ํ•œ ํ˜ธ์ŠคํŠธ์™€ ์ •์ฑ…์—์„œ release `0.1.0-5daee5d-b3fa569e`(Git tag๊ฐ€ ์•„๋‹Œ release ID)์˜ macOS SLO qualification์„ ๋งˆ์ณค์œผ๋ฉฐ Linux ๊ฐ•์ œ ๋ณดํ˜ธ๋Š” qualificationํ•˜์ง€ ์•Š์•˜๋‹ค. CodeSpace ์†Œ๋น„(CS-RG)๋Š” ์ง„ํ–‰ ์ค‘์ด๋ฉฐ qualification์„ ๋งˆ์น˜์ง€ ์•Š์•˜๋‹ค. CSRG-U1์€ opt-in ์ƒํƒœ ์ „์šฉ ์—ฐ๊ฒฐ์„ ์ถ”๊ฐ€ํ–ˆ๋‹ค. `devguard` feature๋กœ ๋นŒ๋“œํ•˜๊ณ  `--devguard status`๋กœ ์‹œ์ž‘ํ•œ gateway๋Š” [`6e7e065`](upstream-lock.md#devguard-client-pin)์— ๊ณ ์ •ํ•œ DevGuard client๋กœ DevGuard์˜ ์ƒํƒœ๋ฅผ `workspace_info`์— ๋ณด๊ณ ํ•œ๋‹ค. CSRG-U2๋ถ€ํ„ฐ๋Š” CodeSpace์˜ ์‹คํ–‰์„ ์†Œ์œ ํ•œ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์Šค์Šค๋กœ๋ฅผ DevGuard์— ๋“ฑ๋กํ•˜๋ฉฐ(`--devguard register`), workspace์˜ `resources` ์„ค์ •์œผ๋กœ ์ž์› ์ฐธ์—ฌ๋ฅผ ํ•„์ˆ˜๋กœ ์ง€์ •ํ•  ์ˆ˜ ์žˆ๋‹ค. CSRG-U3๋ถ€ํ„ฐ ๊ทธ ์†Œ์œ ์ž๋Š” ์ด๋Ÿฐ ์‹คํ–‰๋งˆ๋‹ค ๋ฌด์—‡์„ ๋งŒ๋“ค๊ธฐ ์ „์— ํ”„๋กœ์„ธ์Šค slot์„ ์žก๊ณ  DevGuard์˜ ์‹คํ–‰ ํ—ˆ์šฉ์„ ๋ฐ›๋Š”๋‹ค. CSRG-U4๋ถ€ํ„ฐ๋Š” launch helper๋ฅผ ์„ค์ •ํ–ˆ์œผ๋ฉด(`--devguard-launch-helper`) ๊ทธ ์‹คํ–‰์„ DevGuard์˜ launch helper๋กœ ํ•œ ๋ฒˆ ์‹คํ–‰ํ•œ๋‹ค. DevGuard๋Š” helper์˜ scope๋ฅผ ๋ฌถ๊ณ  ์ •์ฑ…์„ ์ ์šฉํ•œ ๋’ค์— ์‹คํ–‰์„ ์Šน์ธํ•œ๋‹ค. helper๊ฐ€ ์—†์œผ๋ฉด ์‹œ์ž‘ํ•˜์ง€ ์•Š์€ attempt๋ฅผ ์ทจ์†Œํ•˜๊ณ  ์‹คํ–‰์„ ๊ฑฐ๋ถ€ํ•œ๋‹ค. DevGuard ์—†์ด ์‹คํ–‰ํ•˜๋Š” ๊ฒƒ์€ ์—†๋‹ค. CSRG-U5๋ถ€ํ„ฐ ํ”„๋กœ์„ธ์Šค๋งˆ๋‹ค reaper๋Š” ํ•˜๋‚˜๋‹ค. DevGuard๋Š” ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์˜ scope๋ฅผ root๊ฐ€ reap๋˜๊ธฐ ์ „์— ๊ด€์ฐฐํ•œ๋‹ค. pipe์—์„œ๋Š” CodeSpace ์ž์‹ ์˜ reaper๊ฐ€, PTY์—์„œ๋Š” [๊ธฐ๋ก๋œ ํŒจ์น˜](upstream-lock.md#codex-downstream-patch)๋ฅผ ์ ์šฉํ•œ ๊ณ ์ • Codex PTY์˜ waiter๊ฐ€ ๊ทธ ๊ธฐํšŒ๋ฅผ ์ค€๋‹ค. ๊ด€๋ฆฌ ๋Œ€์ƒ ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฃฝ์ด๋ฉด ๊ทธ scope๋„ ์ข…๋ฃŒํ•˜๋ฉฐ, ์‹œ์ž‘ํ•˜์ง€ ์•Š์•˜์Œ์ด ์ฆ๋ช…๋œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์˜ ์Šน์ธ์€ ๋‹ค์‹œ ์žฌ๊ฐœํ•  ์ˆ˜ ์žˆ๋‹ค. qualification์€ CSRG-U6์ด ๋งก์œผ๋ฉฐ, ๊ทธ ์ ˆ์ฐจ์™€ ํ•˜๋„ค์Šค๋Š” [CS-RG qualification](cs-rg-qualification.md)์— ์žˆ๋‹ค. ์•„์ง qualification์„ ๋งˆ์นœ ์กฐํ•ฉ์€ ์—†๋‹ค. qualification์„ ๋งˆ์นœ release๋Š” ์‹คํ–‰์— ์“ธ ํ›„๋ณด๋กœ ๋‚จ์œผ๋ฉฐ ์„ ํƒ๋œ pin์ด ์•„๋‹ˆ๋‹ค. client pin์€ client crate์˜ ์†Œ์Šค pin์ด๋‹ค. DevGuard [์„ค๊ณ„ ๊ฐœ์ • 1](https://github.com/novelKR/DevGuard/blob/d4981b4c241cff42687f5c2c681b583c7847776e/docs/ko/design-revision-1.md)์€ [DevGuard PR #8](https://github.com/novelKR/DevGuard/pull/8)์˜ ๋ณ‘ํ•ฉ `d4981b4c241cff42687f5c2c681b583c7847776e`๋กœ ๋ฐ˜์˜๋˜์—ˆ์œผ๋ฉฐ CS-RG์˜ ์‹คํ–‰ ๊ณ„์ธต๊ณผ ์ž‘์—… ์ˆœ์„œ๋ฅผ ๊ฐœ์ •ํ–ˆ๋‹ค. ์ด ๊ฐœ์ •์€ ๊ณ„ํš๊ณผ ์ดํ›„ ๊ตฌํ˜„์„ ๊ตฌ์†ํ•˜๋Š” ์„ค๊ณ„ ๊ฒฐ์ •์„ ๋ฐ”๊พผ๋‹ค. ํ˜„์žฌ CodeSpace ๋™์ž‘์ด๋‚˜ ๊ตฌํ˜„ยทqualification ์ƒํƒœ๋Š” ๋ฐ”๊พธ์ง€ ์•Š๋Š”๋‹ค. @@ -267,7 +267,7 @@ CSRG-U5๋Š” ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์˜ ์‹คํ–‰ ๋’ค ์ƒ๋ช…์ฃผ๊ธฐ๋ฅผ ์™„์„ฑํ•œ๋‹ค. ๋ˆ„ | CSRG-P4 | C07, C09 | ๋ชจ๋“œ ์ „๋ฐ˜์˜ ๋™๋“ฑ์„ฑยท์žฅ์•  ์‹œํ—˜, ์ด์–ด์„œ backend ์ˆ˜๋ ด ๊ฒฐ์ • | | CSRG-P5 | C08 | C09๊ฐ€ ๋‚จ๊ธด head์˜ qualification | -CSRG-C09๋Š” ์ตœ์ข… qualification ์ „์— ๋ฐ˜๋“œ์‹œ ๋‚ด๋ ค์•ผ ํ•˜๋Š” ๊ฒฐ์ •์ด๋‹ค. ๊ธฐ์กด `off` backend๋ฅผ ํ†ตํ•ฉํ•˜๊ณ  ๋Œ€์ฒด๋œ ์ฝ”๋“œยท๋ถ„๊ธฐยทfixtureยท์˜์กด์„ฑ์„ ์ œ๊ฑฐํ•˜๊ฑฐ๋‚˜, ์ด์œ ยท๋ฒ”์œ„ยท์ค‘๋ณต ๋ถ€๋ถ„ยท์ œ๊ณตํ•˜์ง€ ์•Š๋Š” capabilityยท์žฌ๊ฒ€ํ†  ์‹œ์ ยท์ œ๊ฑฐ ๊ธฐ์ค€์„ ๊ธฐ๋กํ•œ ์ œํ•œ์  ํ˜ธํ™˜ backend๋กœ ์œ ์ง€ํ•œ๋‹ค. ๋ฌธ์„œ์— "๊ฒ€ํ† ํ•จ"์ด๋ผ๊ณ  ์ ๋Š” ๊ฒƒ๋งŒ์œผ๋กœ๋Š” ์™„๋ฃŒ๋˜์ง€ ์•Š๋Š”๋‹ค. C09๊ฐ€ ์ฝ”๋“œ๋ฅผ ๋ฐ”๊พธ๋ฉด ์˜ํ–ฅ๋ฐ›๋Š” C07 ๋™๋“ฑ์„ฑ ์‹œํ—˜์„ ๋‹ค์‹œ ์‹คํ–‰ํ•˜๊ณ , CSRG-C08์€ ๊ทธ ๊ฒฐ๊ณผ head๋ฅผ qualificationํ•œ๋‹ค. +CSRG-C09๋Š” ์ตœ์ข… qualification ์ „์— ๋ฐ˜๋“œ์‹œ ๋‚ด๋ ค์•ผ ํ•˜๋Š” ๊ฒฐ์ •์ด๋‹ค. ๊ธฐ์กด `off` backend๋ฅผ ํ†ตํ•ฉํ•˜๊ณ  ๋Œ€์ฒด๋œ ์ฝ”๋“œยท๋ถ„๊ธฐยทfixtureยท์˜์กด์„ฑ์„ ์ œ๊ฑฐํ•˜๊ฑฐ๋‚˜, ์ด์œ ยท๋ฒ”์œ„ยท์ค‘๋ณต ๋ถ€๋ถ„ยท์ œ๊ณตํ•˜์ง€ ์•Š๋Š” capabilityยท์žฌ๊ฒ€ํ†  ์‹œ์ ยท์ œ๊ฑฐ ๊ธฐ์ค€์„ ๊ธฐ๋กํ•œ ์ œํ•œ์  ํ˜ธํ™˜ backend๋กœ ์œ ์ง€ํ•œ๋‹ค. ๋ฌธ์„œ์— "๊ฒ€ํ† ํ•จ"์ด๋ผ๊ณ  ์ ๋Š” ๊ฒƒ๋งŒ์œผ๋กœ๋Š” ์™„๋ฃŒ๋˜์ง€ ์•Š๋Š”๋‹ค. C09๊ฐ€ ์ฝ”๋“œ๋ฅผ ๋ฐ”๊พธ๋ฉด ์˜ํ–ฅ๋ฐ›๋Š” C07 ๋™๋“ฑ์„ฑ ์‹œํ—˜์„ ๋‹ค์‹œ ์‹คํ–‰ํ•˜๊ณ , CSRG-C08์€ ๊ทธ ๊ฒฐ๊ณผ head๋ฅผ qualificationํ•œ๋‹ค. CS-RG๋Š” ์ฝ”๋“œ๋ฅผ ๋ฐ”๊พธ์ง€ ์•Š๊ณ  `off`๋ฅผ ํ˜ธํ™˜ ๊ฒฝ๋กœ๋กœ ์œ ์ง€ํ•˜๋ฉฐ, ๊ทธ ๊ธฐ๋ก์€ [๊ธฐ์กด `off` ๊ฒฝ๋กœ ๊ฒฐ์ •](cs-rg-qualification.md#legacy-off-path)์— ์žˆ๋‹ค. ์ด ๋Œ€์‘ ๋ฌธ์„œ๋Š” DevGuard [PR ์ง„ํ–‰์„œ](https://github.com/novelKR/DevGuard/blob/d4981b4c241cff42687f5c2c681b583c7847776e/docs/ko/planning/pr-delivery.md)์˜ CSP-D04์ด๋ฉฐ CSRG-P0 ์‹œ์ž‘ ์ „์— ์ „๋‹ฌํ•ด์•ผ ํ•˜๋Š” ์กฐ๊ฑด์ด๋‹ค. ๊ณ„ํšํ•œ ๊ตฌ์กฐ๋Š” [์•„ํ‚คํ…์ฒ˜](architecture.md), ์‹คํ–‰ ๊ณ„์•ฝ์€ [์‹คํ–‰ ๊ณ„์•ฝ](execution-substrate.md), ์žฌ์‚ฌ์šฉ ๊ฒฐ์ •์€ [Codex ์žฌ์‚ฌ์šฉ ๋ฒ”์œ„](codex-reuse.md), ์˜์กด์„ฑยทCI ๊ฒฝ๊ณ„๋Š” [์—…์ŠคํŠธ๋ฆผ ์—…๋ฐ์ดํŠธ](upstream-update.md)์— ์ •๋ฆฌํ•œ๋‹ค. diff --git a/docs/translations.json b/docs/translations.json index 7696d70..97812a7 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -75,8 +75,63 @@ "์ตœ์†Œ-๋„์ž…-์กฐ๊ฑด", "์ตœ์ดˆ-gateway-๋ณต๊ตฌ-๋ฒ”์œ„" ], - "source_sha256": "b08b20641a0bbd91a9635e31cf5051171d7844d5ae19c8c5d5ce13b30dfe2228", - "translation_sha256": "34927ccff3fd854b9041e202f2a807cd89ac3eb65e632598e9fed3a02fefe11c" + "source_sha256": "7ca40059fa574da9d60492618d2520c1e4a8d6a97e591d55d3b3c61e1044c76b", + "translation_sha256": "5fba264f1c5e64d56b997a80078876abc5d863580f0c7f49184bc75b70f7f873" + }, + { + "id": "cs-rg-qualification", + "section": "project", + "order": 2, + "route": "/guide/cs-rg-qualification", + "source": "docs/cs-rg-qualification.md", + "translation": "docs/ko/cs-rg-qualification.md", + "anchors": [ + "accounting", + "authority-failure", + "authority-์žฅ์• ", + "candidate", + "control-intervals", + "control-load", + "control-path", + "control-path-availability", + "control-verdicts", + "coverage-of-the-orders-fault-list", + "cs-rg-qualification", + "cs-rg-qualification-protocol", + "cs-rg-qualification-์ ˆ์ฐจ", + "evidence", + "fault-cases", + "fault-coverage", + "functional-matrix", + "intervals-and-samples", + "legacy-off-path", + "legacy-off-path-decision", + "load", + "regression-and-parity", + "regressions", + "resource-accounting", + "running", + "running-the-harness", + "supported-modes", + "the-candidate", + "verdicts", + "๊ตฌ๊ฐ„๊ณผ-ํ‘œ๋ณธ", + "๊ธฐ๋Šฅ-ํ–‰๋ ฌ", + "๊ธฐ์กด-off-๊ฒฝ๋กœ-๊ฒฐ์ •", + "๋ถ€ํ•˜", + "์ž์›-ํšŒ๊ณ„", + "์ž‘์—…-์ง€์‹œ์„œ์˜-์žฅ์• -๋ชฉ๋ก๊ณผ์˜-๋Œ€์‘", + "์žฅ์• -์‚ฌ๋ก€", + "์ œ์–ด-๊ฒฝ๋กœ-๊ฐ€์šฉ์„ฑ", + "์ฆ๊ฑฐ", + "์ง€์›-๋ชจ๋“œ", + "ํŒ์ •", + "ํ•˜๋„ค์Šค-์‹คํ–‰", + "ํšŒ๊ท€์™€-๋™๋“ฑ์„ฑ", + "ํ›„๋ณด-์กฐํ•ฉ" + ], + "source_sha256": "018ad1ba83f5f63c2442d2d76b818ecee88742e5698404f3ef1d6c40d67fe0ac", + "translation_sha256": "1cdf7e966d26809d9a35689a12489a9444322c46d497d906e795e0b842175076" }, { "id": "overview", From 7a0a748bb0914b1fead53f865ad33b9e672a98c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 17:06:42 +0000 Subject: [PATCH 3/4] policy: say that required executions are launched through the authority The comment on Participation::Required still said CodeSpace could not launch them yet, which stopped being true with CSRG-U4. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/policy/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/policy/src/lib.rs b/crates/policy/src/lib.rs index 99550da..de85b39 100644 --- a/crates/policy/src/lib.rs +++ b/crates/policy/src/lib.rs @@ -60,9 +60,8 @@ pub enum Participation { /// Executions run as they always have, without a resource authority. #[default] Off, - /// Every new execution must be admitted and launched through the resource authority. - /// CodeSpace admits it (CSRG-U3) but cannot launch it there yet, so none runs; none runs - /// without the authority either. + /// Every new execution must be admitted (CSRG-U3) and launched (CSRG-U4) through the + /// resource authority; none runs without it. Required, } From 8ed8b0839137cc943727fd15ee69f40d52612538 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 10:37:37 +0000 Subject: [PATCH 4/4] scripts: fix the three harness findings of the macOS check (CSRG-U6) The qualification Mac ran the suites at 7a0a748 on 2026-10-13. Three of its failures were the harness's: - matrix expected a governed PTY's death by SIGTERM as exit code 1. A governed PTY child inherits its launch descriptors, so the pinned Codex PTY spawns it on its descriptor-keeping path, which reports 128 plus the signal (143), as docs/devguard-integration.md already says. The harness, the protocol and docs/agent-integration.md, which named only 1, now say so. - matrix's settled check read DevGuard's instances once, right after the last gateway exited, and failed on an instance DevGuard retired moments later with nothing charged. It now awaits the charge and the instances for up to 60 seconds, and its stop message names both. - authority's restarted phase started its termination targets while DevGuard still kept new work closed after the stalled phase's stale host sample. A target refused RESOURCE_UNAVAILABLE is now tried again every second for up to 90 seconds, and the phase records the wait. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/agent-integration.md | 2 +- docs/cs-rg-qualification.md | 8 ++-- docs/ko/agent-integration.md | 2 +- docs/ko/cs-rg-qualification.md | 8 ++-- docs/translations.json | 8 ++-- scripts/qualify-devguard.py | 60 +++++++++++++++++++------- scripts/tests/test_qualify_devguard.py | 40 +++++++++++++++++ 7 files changed, 98 insertions(+), 30 deletions(-) diff --git a/docs/agent-integration.md b/docs/agent-integration.md index 997c946..de9bcde 100644 --- a/docs/agent-integration.md +++ b/docs/agent-integration.md @@ -85,7 +85,7 @@ The response contains a server-issued `process_id` and `dispatch_status`. `confi } ``` -Each result has `chunk`, `cursor`, `eof`, `output_lost`, and `retained_from`. Output combines stdout/stderr without preserving their identity. EOF means output collection is complete; it is not a successful exit status. Judge termination with `process_status`: `state` is `running` or `exited`. After exit, `termination` is `exited`, `signaled`, `timeout`, `terminated`, or `unknown`. `exit_code` is present only when `termination` is `exited`. `signaled` means that a signal CodeSpace did not send ended the process, for example a crash or a kill from another process. `signal` then gives its `number` and, when known, its `name`, such as `SIGKILL`. Signal numbers differ between operating systems. A kill that CodeSpace requested is reported as `timeout` or `terminated` instead. `exited` does not rule out a signal: a `tty: true` process reports a signal death as `exited` with `exit_code` 1, the same as a command that failed with status 1, and under the Linux command sandbox the status is that of the sandbox helper ([runner isolation](runner-isolation.md)). `signaled`, `timeout`, `terminated`, and `unknown` are not success, even when `eof` is true. The last 256 KiB are retained. If `output_lost` is true, the retained window is not the complete log. Do not claim a build or test passed solely from EOF or an incomplete log. If success cannot be established from a reliable task-specific result, report it as unverified. +Each result has `chunk`, `cursor`, `eof`, `output_lost`, and `retained_from`. Output combines stdout/stderr without preserving their identity. EOF means output collection is complete; it is not a successful exit status. Judge termination with `process_status`: `state` is `running` or `exited`. After exit, `termination` is `exited`, `signaled`, `timeout`, `terminated`, or `unknown`. `exit_code` is present only when `termination` is `exited`. `signaled` means that a signal CodeSpace did not send ended the process, for example a crash or a kill from another process. `signal` then gives its `number` and, when known, its `name`, such as `SIGKILL`. Signal numbers differ between operating systems. A kill that CodeSpace requested is reported as `timeout` or `terminated` instead. `exited` does not rule out a signal: a `tty: true` process reports a signal death as `exited`, with `exit_code` 1, the same as a command that failed with status 1, or, when CodeSpace launches it through DevGuard (participation `required`), with 128 plus the signal number, such as 143 for SIGTERM. Under the Linux command sandbox the status is that of the sandbox helper ([runner isolation](runner-isolation.md)). `signaled`, `timeout`, `terminated`, and `unknown` are not success, even when `eof` is true. The last 256 KiB are retained. If `output_lost` is true, the retained window is not the complete log. Do not claim a build or test passed solely from EOF or an incomplete log. If success cannot be established from a reliable task-specific result, report it as unverified. ```json { diff --git a/docs/cs-rg-qualification.md b/docs/cs-rg-qualification.md index bad91aa..6638092 100644 --- a/docs/cs-rg-qualification.md +++ b/docs/cs-rg-qualification.md @@ -82,7 +82,7 @@ The harness talks to `codespace-mcp` over standard input and output as any MCP c | Scenario | Pipe and PTY | Passes when | | --- | --- | --- | | registration | | `workspace_info` reports participation `registration`, the authority available, the owner (`in_process` or `worker`) registered, governing executions, and DevGuard's one active `codespace` instance is that process | -| exits | each | exit 0 and exit 3 are reported as exited with their codes; a SIGTERM death is `signaled` with signal 15 on a pipe, and exited with code 1 on a PTY, as the pinned Codex PTY reports it ([process status](agent-integration.md)) | +| exits | each | exit 0 and exit 3 are reported as exited with their codes; a SIGTERM death is `signaled` with signal 15 on a pipe, and exited with code 143 (128 + 15) on a PTY, as the pinned Codex PTY reports a governed child, which inherits its launch descriptors ([process status](agent-integration.md)) | | io | each | on a pipe, `write_stdin` reaches `cat` and its output returns, `process_resize` is refused `PROCESS_NOT_TTY` and termination is reported; on a PTY, the terminal starts at 24ร—80, a resize to 40ร—100 is answered and seen by the process, and stdin reaches it | | termination | each | `terminate_process` answers, the process is reported terminated and its scope's process group is gone | | timeout | each | a `sleep 600` is reported `timeout` | @@ -94,7 +94,7 @@ The harness talks to `codespace-mcp` over standard input and output as any MCP c | status-mode | | under `--devguard status` a governed execution is refused `RESOURCE_POLICY_UNSUPPORTED` before an attempt exists; `off` still runs | | authority-unavailable | | with no authority at the socket, it is refused `RESOURCE_AUTHORITY_UNAVAILABLE`, DevGuard holds no attempt for it, and `off` still runs | | runner-socket | `uds` only | with a worker the gateway did not start, governed executions are refused `RESOURCE_POLICY_UNSUPPORTED` before an attempt exists | -| settled | | nothing of the consumer stays charged and no instance stays active; otherwise the matrix stops | +| settled | | nothing of the consumer stays charged and no instance stays active, both awaited for up to 60 seconds because DevGuard retires an instance once it sees that its owner is gone; otherwise the matrix stops | Every governed launch is then checked against DevGuard's own record of its attempt ([accounting](#accounting)). @@ -188,7 +188,7 @@ A repetition passes when both intervals pass and the service still runs the rele ### Authority failure -`authority` measures process control while the fixture authority is unhealthy, in five 15-second phases per runner mode, against the same running status targets: `baseline`; `paused` (the reconciler paused); `stalled` (its lock held for the whole phase); `restarted`; `stopped` (its process killed). Status is sampled every second, the I/O echo every 3 seconds, three governed targets started before the fault are terminated at 3, 7 and 11 seconds, and a new governed execution is attempted at 1, 5, 9 and 13 seconds. Each phase is judged by the interval rules above, and new executions must be admitted while the authority can answer (`baseline`, `paused`, `restarted`) and refused `RESOURCE_AUTHORITY_UNAVAILABLE` or `ADMISSION_UNKNOWN` while it cannot (`stalled`, `stopped`). After a restart or once stopped, DevGuard cannot release targets started before the fault: an unreleased target is reported there, not failed. +`authority` measures process control while the fixture authority is unhealthy, in five 15-second phases per runner mode, against the same running status targets: `baseline`; `paused` (the reconciler paused); `stalled` (its lock held for the whole phase); `restarted`; `stopped` (its process killed). Status is sampled every second, the I/O echo every 3 seconds, three governed targets started before the fault are terminated at 3, 7 and 11 seconds, and a new governed execution is attempted at 1, 5, 9 and 13 seconds. The targets are started once DevGuard admits them: a host sample that went stale while the authority was held keeps new work closed until its samples have been clear for 30 seconds, so after `stalled` a target refused `RESOURCE_UNAVAILABLE` is tried again every second for up to 90 seconds, and the phase records how long that took. Each phase is judged by the interval rules above, and new executions must be admitted while the authority can answer (`baseline`, `paused`, `restarted`) and refused `RESOURCE_AUTHORITY_UNAVAILABLE` or `ADMISSION_UNKNOWN` while it cannot (`stalled`, `stopped`). After a restart or once stopped, DevGuard cannot release targets started before the fault: an unreleased target is reported there, not failed. @@ -220,7 +220,7 @@ This is the decision record that the work order requires for the existing backen - **Scope.** Participation `off`, the default and what a missing setting means, on every platform and runner mode, pipe and PTY. Its executions are the request's own command: pipe children have CodeSpace's single reaper, which governed executions share; PTY children use the pinned Codex PTY spawn without the observation window before the reap. - **Why.** The work order does not migrate code for neatness alone. `off` is the product default; `required` is an operator's explicit choice, unsupported on Linux until DG-LINUX, so `off` is the only path there. Making governed launching the only path would remove every Linux execution and every setup without DevGuard. -- **What is duplicated.** One spawn path serves both and chooses the program: the request's command, or DevGuard's launch helper with its permit and report descriptors (`How::Direct` and `How::Managed` in [`crates/runner/src/process.rs`](../crates/runner/src/process.rs)). Reaping is not duplicated: a pipe child of either kind has CodeSpace's one reaper, and a PTY child the pinned Codex PTY's waiter. A PTY differs in one call, `spawn_inheriting` for ungoverned executions and `spawn_observed`, from the [recorded patch](upstream-patches.md), for governed ones. The governed path adds admission, launch and settlement ([`crates/runner/src/launch.rs`](../crates/runner/src/launch.rs)), which `off` never enters. Without the `devguard` feature no DevGuard package is in the build; with it, only DevGuard's client and contract crates are, as the `dependencies` stage checks. +- **What is duplicated.** One spawn path serves both and chooses the program: the request's command, or DevGuard's launch helper with its permit and report descriptors (`How::Direct` and `How::Managed` in [`crates/runner/src/process.rs`](../crates/runner/src/process.rs)). Reaping is not duplicated: a pipe child of either kind has CodeSpace's one reaper, and a PTY child the pinned Codex PTY's waiter. A PTY differs in one call, `spawn_inheriting` for ungoverned executions and `spawn_observed`, from the [recorded patch](upstream-patches.md), for governed ones. A governed PTY child also inherits its launch descriptors, so the pinned Codex PTY spawns it on its descriptor-keeping path, which reports a signal death as exit code 128 plus the signal where an ungoverned PTY reports 1 ([process status](agent-integration.md)). The governed path adds admission, launch and settlement ([`crates/runner/src/launch.rs`](../crates/runner/src/launch.rs)), which `off` never enters. Without the `devguard` feature no DevGuard package is in the build; with it, only DevGuard's client and contract crates are, as the `dependencies` stage checks. - **What `off` does not provide.** No admission, reservation or accounting at DevGuard, no scope policing, and no observation of an ungoverned PTY root before its reap. Only `off` executions can use the Linux command sandbox; governed ones are refused where it is on. - **Revisit.** When DG-LINUX qualifies Linux enforcement, when a Codex pin move changes PTY spawning, or when `required` becomes the default. - **Removal.** Only with an independent product reason and regression evidence that every supported mode keeps its behavior without it. diff --git a/docs/ko/agent-integration.md b/docs/ko/agent-integration.md index b242971..f91cadc 100644 --- a/docs/ko/agent-integration.md +++ b/docs/ko/agent-integration.md @@ -85,7 +85,7 @@ MCP ํด๋ผ์ด์–ธํŠธ SDK๋กœ stdio ๋˜๋Š” Streamable HTTP๋ฅผ ์ดˆ๊ธฐํ™”ํ•˜๊ณ , ์ดˆ } ``` -๊ฒฐ๊ณผ์—๋Š” `chunk`, `cursor`, `eof`, `output_lost`, `retained_from`์ด ์žˆ์Šต๋‹ˆ๋‹ค. stdout/stderr๋Š” ๊ตฌ๋ถ„ ์—†์ด ํ•ฉ์ณ์ง‘๋‹ˆ๋‹ค. EOF๋Š” ์ถœ๋ ฅ ์ˆ˜์ง‘์ด ๋๋‚ฌ๋‹ค๋Š” ๋œป์ด๋ฉฐ ์„ฑ๊ณต ์ข…๋ฃŒ๋ฅผ ๋‚˜ํƒ€๋‚ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ข…๋ฃŒ๋Š” `process_status`๋กœ ํŒ์ •ํ•ฉ๋‹ˆ๋‹ค. `state`๋Š” `running` ๋˜๋Š” `exited`์ž…๋‹ˆ๋‹ค. ์ข…๋ฃŒ ํ›„ `termination`์€ `exited`ยท`signaled`ยท`timeout`ยท`terminated`ยท`unknown` ์ค‘ ํ•˜๋‚˜์ž…๋‹ˆ๋‹ค. `exit_code`๋Š” `termination`์ด `exited`์ผ ๋•Œ๋งŒ ์žˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. `signaled`๋Š” CodeSpace๊ฐ€ ๋ณด๋‚ด์ง€ ์•Š์€ ์‹ ํ˜ธ๋กœ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚ฌ๋‹ค๋Š” ๋œป์ž…๋‹ˆ๋‹ค. ํฌ๋ž˜์‹œ๋‚˜ ๋‹ค๋ฅธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋ณด๋‚ธ kill์ด ์—ฌ๊ธฐ์— ํ•ด๋‹นํ•ฉ๋‹ˆ๋‹ค. ์ด๋•Œ `signal`์—๋Š” ์‹ ํ˜ธ ๋ฒˆํ˜ธ `number`์™€, ์•Œ ์ˆ˜ ์žˆ๋Š” ๊ฒฝ์šฐ `SIGKILL` ๊ฐ™์€ ์ด๋ฆ„ `name`์ด ๋“ค์–ด๊ฐ‘๋‹ˆ๋‹ค. ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋Š” ์šด์˜์ฒด์ œ๋งˆ๋‹ค ๋‹ค๋ฆ…๋‹ˆ๋‹ค. CodeSpace๊ฐ€ ์š”์ฒญํ•œ kill์€ `signaled`๊ฐ€ ์•„๋‹ˆ๋ผ `timeout`์ด๋‚˜ `terminated`๋กœ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค. `exited`๋ผ๊ณ  ํ•ด์„œ ์‹ ํ˜ธ๋กœ ๋๋‚˜์ง€ ์•Š์•˜๋‹ค๋Š” ๋œป์€ ์•„๋‹™๋‹ˆ๋‹ค. `tty: true` ํ”„๋กœ์„ธ์Šค๋Š” ์‹ ํ˜ธ๋กœ ๋๋‚˜๋„ `exited`์™€ `exit_code` 1๋กœ ๋ณด๊ณ ๋˜์–ด ์ƒํƒœ 1๋กœ ์‹คํŒจํ•œ ๋ช…๋ น๊ณผ ๊ตฌ๋ณ„๋˜์ง€ ์•Š๊ณ , Linux ๋ช…๋ น ์ƒŒ๋“œ๋ฐ•์Šค์—์„œ๋Š” ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ์˜ ์ข…๋ฃŒ ์ƒํƒœ๊ฐ€ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค([๋Ÿฌ๋„ˆ ๊ฒฉ๋ฆฌ](runner-isolation.md)). `signaled`ยท`timeout`ยท`terminated`ยท`unknown`์€ `eof`๊ฐ€ ์ฐธ์ด์–ด๋„ ์„ฑ๊ณต์ด ์•„๋‹™๋‹ˆ๋‹ค. ๋งˆ์ง€๋ง‰ 256 KiB๋งŒ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. `output_lost`๊ฐ€ ์ฐธ์ด๋ฉด ๋ณด๊ด€ ์ฐฝ์ด ์ „์ฒด ๋กœ๊ทธ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. EOF๋‚˜ ๋ถˆ์™„์ „ํ•œ ๋กœ๊ทธ๋งŒ์œผ๋กœ ๋นŒ๋“œยทํ…Œ์ŠคํŠธ ์„ฑ๊ณต์„ ์„ ์–ธํ•˜์ง€ ๋งˆ์„ธ์š”. ์‹ ๋ขฐํ•  ์ˆ˜ ์žˆ๋Š” ์ž‘์—…๋ณ„ ๊ฒฐ๊ณผ๋กœ ์„ฑ๊ณต์„ ํ™•์ธํ•  ์ˆ˜ ์—†๋‹ค๋ฉด ๋ฏธ๊ฒ€์ฆ์œผ๋กœ ๋ณด๊ณ ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. +๊ฒฐ๊ณผ์—๋Š” `chunk`, `cursor`, `eof`, `output_lost`, `retained_from`์ด ์žˆ์Šต๋‹ˆ๋‹ค. stdout/stderr๋Š” ๊ตฌ๋ถ„ ์—†์ด ํ•ฉ์ณ์ง‘๋‹ˆ๋‹ค. EOF๋Š” ์ถœ๋ ฅ ์ˆ˜์ง‘์ด ๋๋‚ฌ๋‹ค๋Š” ๋œป์ด๋ฉฐ ์„ฑ๊ณต ์ข…๋ฃŒ๋ฅผ ๋‚˜ํƒ€๋‚ด์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ข…๋ฃŒ๋Š” `process_status`๋กœ ํŒ์ •ํ•ฉ๋‹ˆ๋‹ค. `state`๋Š” `running` ๋˜๋Š” `exited`์ž…๋‹ˆ๋‹ค. ์ข…๋ฃŒ ํ›„ `termination`์€ `exited`ยท`signaled`ยท`timeout`ยท`terminated`ยท`unknown` ์ค‘ ํ•˜๋‚˜์ž…๋‹ˆ๋‹ค. `exit_code`๋Š” `termination`์ด `exited`์ผ ๋•Œ๋งŒ ์žˆ์„ ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค. `signaled`๋Š” CodeSpace๊ฐ€ ๋ณด๋‚ด์ง€ ์•Š์€ ์‹ ํ˜ธ๋กœ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋๋‚ฌ๋‹ค๋Š” ๋œป์ž…๋‹ˆ๋‹ค. ํฌ๋ž˜์‹œ๋‚˜ ๋‹ค๋ฅธ ํ”„๋กœ์„ธ์Šค๊ฐ€ ๋ณด๋‚ธ kill์ด ์—ฌ๊ธฐ์— ํ•ด๋‹นํ•ฉ๋‹ˆ๋‹ค. ์ด๋•Œ `signal`์—๋Š” ์‹ ํ˜ธ ๋ฒˆํ˜ธ `number`์™€, ์•Œ ์ˆ˜ ์žˆ๋Š” ๊ฒฝ์šฐ `SIGKILL` ๊ฐ™์€ ์ด๋ฆ„ `name`์ด ๋“ค์–ด๊ฐ‘๋‹ˆ๋‹ค. ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋Š” ์šด์˜์ฒด์ œ๋งˆ๋‹ค ๋‹ค๋ฆ…๋‹ˆ๋‹ค. CodeSpace๊ฐ€ ์š”์ฒญํ•œ kill์€ `signaled`๊ฐ€ ์•„๋‹ˆ๋ผ `timeout`์ด๋‚˜ `terminated`๋กœ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค. `exited`๋ผ๊ณ  ํ•ด์„œ ์‹ ํ˜ธ๋กœ ๋๋‚˜์ง€ ์•Š์•˜๋‹ค๋Š” ๋œป์€ ์•„๋‹™๋‹ˆ๋‹ค. `tty: true` ํ”„๋กœ์„ธ์Šค๋Š” ์‹ ํ˜ธ๋กœ ๋๋‚˜๋„ `exited`๋กœ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค. `exit_code`๋Š” 1์ด์–ด์„œ ์ƒํƒœ 1๋กœ ์‹คํŒจํ•œ ๋ช…๋ น๊ณผ ๊ตฌ๋ณ„๋˜์ง€ ์•Š๊ณ , CodeSpace๊ฐ€ DevGuard๋ฅผ ๊ฑฐ์ณ ๋„์šด ํ”„๋กœ์„ธ์Šค(participation `required`)๋ผ๋ฉด 128์— ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋ฅผ ๋”ํ•œ ๊ฐ’(SIGTERM์ด๋ฉด 143)์ž…๋‹ˆ๋‹ค. Linux ๋ช…๋ น ์ƒŒ๋“œ๋ฐ•์Šค์—์„œ๋Š” ์ƒŒ๋“œ๋ฐ•์Šค ๋„์šฐ๋ฏธ์˜ ์ข…๋ฃŒ ์ƒํƒœ๊ฐ€ ๋ณด๊ณ ๋ฉ๋‹ˆ๋‹ค([๋Ÿฌ๋„ˆ ๊ฒฉ๋ฆฌ](runner-isolation.md)). `signaled`ยท`timeout`ยท`terminated`ยท`unknown`์€ `eof`๊ฐ€ ์ฐธ์ด์–ด๋„ ์„ฑ๊ณต์ด ์•„๋‹™๋‹ˆ๋‹ค. ๋งˆ์ง€๋ง‰ 256 KiB๋งŒ ๋ณด๊ด€ํ•ฉ๋‹ˆ๋‹ค. `output_lost`๊ฐ€ ์ฐธ์ด๋ฉด ๋ณด๊ด€ ์ฐฝ์ด ์ „์ฒด ๋กœ๊ทธ๊ฐ€ ์•„๋‹™๋‹ˆ๋‹ค. EOF๋‚˜ ๋ถˆ์™„์ „ํ•œ ๋กœ๊ทธ๋งŒ์œผ๋กœ ๋นŒ๋“œยทํ…Œ์ŠคํŠธ ์„ฑ๊ณต์„ ์„ ์–ธํ•˜์ง€ ๋งˆ์„ธ์š”. ์‹ ๋ขฐํ•  ์ˆ˜ ์žˆ๋Š” ์ž‘์—…๋ณ„ ๊ฒฐ๊ณผ๋กœ ์„ฑ๊ณต์„ ํ™•์ธํ•  ์ˆ˜ ์—†๋‹ค๋ฉด ๋ฏธ๊ฒ€์ฆ์œผ๋กœ ๋ณด๊ณ ํ•ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ```json { diff --git a/docs/ko/cs-rg-qualification.md b/docs/ko/cs-rg-qualification.md index 1ae1720..bc8c227 100644 --- a/docs/ko/cs-rg-qualification.md +++ b/docs/ko/cs-rg-qualification.md @@ -82,7 +82,7 @@ env -i HOME="$HOME" TMPDIR="$TMPDIR" PATH="/bin:/usr/bin:/bin:/usr/ | ์‹œ๋‚˜๋ฆฌ์˜ค | pipe์™€ PTY | ํ†ต๊ณผ ์กฐ๊ฑด | | --- | --- | --- | | registration | | `workspace_info`๊ฐ€ ์ฐธ์—ฌ `registration`, authority ์‚ฌ์šฉ ๊ฐ€๋Šฅ, ์†Œ์œ ์ž(`in_process` ๋˜๋Š” `worker`) ๋“ฑ๋ก, ์‹คํ–‰ ๊ด€๋ฆฌ ์ค‘์„ ๋ณด๊ณ ํ•˜๊ณ , DevGuard์˜ ํ™œ์„ฑ `codespace` ์ธ์Šคํ„ด์Šค ํ•˜๋‚˜๊ฐ€ ๊ทธ ํ”„๋กœ์„ธ์Šค์ž„ | -| exits | ๊ฐ๊ฐ | exit 0๊ณผ exit 3์ด ๊ทธ ์ฝ”๋“œ์˜ exited๋กœ ๋ณด๊ณ ๋จ. SIGTERM์œผ๋กœ ์ฃฝ์œผ๋ฉด pipe์—์„œ๋Š” signal 15์˜ `signaled`, PTY์—์„œ๋Š” ๊ณ ์ •๋œ Codex PTY๊ฐ€ ๋ณด๊ณ ํ•˜๋Š” ๋Œ€๋กœ ์ฝ”๋“œ 1์˜ exited([ํ”„๋กœ์„ธ์Šค ์ƒํƒœ](agent-integration.md)) | +| exits | ๊ฐ๊ฐ | exit 0๊ณผ exit 3์ด ๊ทธ ์ฝ”๋“œ์˜ exited๋กœ ๋ณด๊ณ ๋จ. SIGTERM์œผ๋กœ ์ฃฝ์œผ๋ฉด pipe์—์„œ๋Š” signal 15์˜ `signaled`, PTY์—์„œ๋Š” ๊ณ ์ •๋œ Codex PTY๊ฐ€ launch descriptor๋ฅผ ๋ฌผ๋ ค๋ฐ›๋Š” ๊ด€๋ฆฌ ๋Œ€์ƒ ์ž์‹์„ ๋ณด๊ณ ํ•˜๋Š” ๋Œ€๋กœ ์ฝ”๋“œ 143(128 + 15)์˜ exited([ํ”„๋กœ์„ธ์Šค ์ƒํƒœ](agent-integration.md)) | | io | ๊ฐ๊ฐ | pipe์—์„œ๋Š” `write_stdin`์ด `cat`์— ๋‹ฟ๊ณ  ์ถœ๋ ฅ์ด ๋Œ์•„์˜ค๋ฉฐ, `process_resize`๊ฐ€ `PROCESS_NOT_TTY`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , ์ข…๋ฃŒ๊ฐ€ ๋ณด๊ณ ๋จ. PTY์—์„œ๋Š” ํ„ฐ๋ฏธ๋„์ด 24ร—80์œผ๋กœ ์‹œ์ž‘ํ•˜๊ณ , 40ร—100 ํฌ๊ธฐ ์กฐ์ •์— ๋‹ตํ•˜๋ฉฐ ํ”„๋กœ์„ธ์Šค๊ฐ€ ์ƒˆ ํฌ๊ธฐ๋ฅผ ๋ณด๊ณ , stdin์ด ๋‹ฟ์Œ | | termination | ๊ฐ๊ฐ | `terminate_process`๊ฐ€ ๋‹ตํ•˜๊ณ , ํ”„๋กœ์„ธ์Šค๊ฐ€ terminated๋กœ ๋ณด๊ณ ๋˜๋ฉฐ, scope์˜ ํ”„๋กœ์„ธ์Šค ๊ทธ๋ฃน์ด ์‚ฌ๋ผ์ง | | timeout | ๊ฐ๊ฐ | `sleep 600`์ด `timeout`์œผ๋กœ ๋ณด๊ณ ๋จ | @@ -94,7 +94,7 @@ env -i HOME="$HOME" TMPDIR="$TMPDIR" PATH="/bin:/usr/bin:/bin:/usr/ | status-mode | | `--devguard status`์—์„œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ attempt๊ฐ€ ์ƒ๊ธฐ๊ธฐ ์ „์— `RESOURCE_POLICY_UNSUPPORTED`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , `off`๋Š” ๊ณ„์† ์‹คํ–‰๋จ | | authority-unavailable | | socket์— authority๊ฐ€ ์—†์œผ๋ฉด `RESOURCE_AUTHORITY_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€๋˜๊ณ , DevGuard์—๋Š” ๊ทธ attempt๊ฐ€ ์—†์œผ๋ฉฐ, `off`๋Š” ๊ณ„์† ์‹คํ–‰๋จ | | runner-socket | `uds`๋งŒ | gateway๊ฐ€ ์‹œ์ž‘ํ•˜์ง€ ์•Š์€ worker์—์„œ๋Š” ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์ด attempt๊ฐ€ ์ƒ๊ธฐ๊ธฐ ์ „์— `RESOURCE_POLICY_UNSUPPORTED`๋กœ ๊ฑฐ๋ถ€๋จ | -| settled | | ์†Œ๋น„์ž์˜ ์–ด๋–ค attempt๋„ ์ž์›์„ ์ฐจ์ง€ํ•˜์ง€ ์•Š๊ณ  ํ™œ์„ฑ ์ธ์Šคํ„ด์Šค๋„ ์—†์Œ. ์•„๋‹ˆ๋ฉด ํ–‰๋ ฌ์„ ๋ฉˆ์ถค | +| settled | | ์†Œ๋น„์ž์˜ ์–ด๋–ค attempt๋„ ์ž์›์„ ์ฐจ์ง€ํ•˜์ง€ ์•Š๊ณ  ํ™œ์„ฑ ์ธ์Šคํ„ด์Šค๋„ ์—†์Œ. DevGuard๋Š” ์†Œ์œ ์ž๊ฐ€ ์‚ฌ๋ผ์ง„ ๊ฒƒ์„ ๋ณธ ๋’ค์— ์ธ์Šคํ„ด์Šค๋ฅผ `retired`๋กœ ๋ฐ”๊พธ๋ฏ€๋กœ ๋‘˜ ๋‹ค 60์ดˆ๊นŒ์ง€ ๊ธฐ๋‹ค๋ฆผ. ์•„๋‹ˆ๋ฉด ํ–‰๋ ฌ์„ ๋ฉˆ์ถค | ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ ๋ชจ๋‘ DevGuard ์ž์‹ ์˜ attempt ๊ธฐ๋ก๊ณผ ๋‹ค์‹œ ๋Œ€์กฐํ•ฉ๋‹ˆ๋‹ค([์ž์› ํšŒ๊ณ„](#accounting)). @@ -188,7 +188,7 @@ fixture authority๋ฅผ ์“ฐ๋Š” gateway ์‹œํ—˜์€ DevGuard ์‹œํ—˜ ๋ฐ”์ด๋„ˆ๋ฆฌ๊ฐ€ ### authority ์žฅ์•  -`authority`๋Š” fixture authority๊ฐ€ ์ •์ƒ์ด ์•„๋‹ ๋•Œ์˜ ํ”„๋กœ์„ธ์Šค ์ œ์–ด๋ฅผ, runner ๋ชจ๋“œ๋งˆ๋‹ค 15์ดˆ์งœ๋ฆฌ ๋‹ค์„ฏ ๋‹จ๊ณ„๋กœ, ๊ณ„์† ์‹คํ–‰ ์ค‘์ธ ๊ฐ™์€ status ๋Œ€์ƒ์— ๋Œ€ํ•ด ์ธก์ •ํ•ฉ๋‹ˆ๋‹ค. ๋‹จ๊ณ„๋Š” `baseline`, `paused`(reconciler ๋ฉˆ์ถค), `stalled`(๋‹จ๊ณ„ ๋‚ด๋‚ด lock์„ ์žก์Œ), `restarted`, `stopped`(ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฃฝ์ž„)์ž…๋‹ˆ๋‹ค. status๋Š” 1์ดˆ๋งˆ๋‹ค, I/O echo๋Š” 3์ดˆ๋งˆ๋‹ค ์žฌ๊ณ , ์žฅ์•  ์ „์— ์‹œ์ž‘ํ•œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์…‹์„ 3, 7, 11์ดˆ์— ์ข…๋ฃŒํ•˜๋ฉฐ, 1, 5, 9, 13์ดˆ์— ์ƒˆ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์„ ์‹œ๋„ํ•ฉ๋‹ˆ๋‹ค. ๋‹จ๊ณ„๋งˆ๋‹ค ์œ„์˜ ๊ตฌ๊ฐ„ ๊ทœ์น™์œผ๋กœ ํŒ์ •ํ•˜๋ฉฐ, ์ƒˆ ์‹คํ–‰์€ authority๊ฐ€ ๋‹ตํ•  ์ˆ˜ ์žˆ๋Š” ๋™์•ˆ(`baseline`, `paused`, `restarted`)์—๋Š” ํ—ˆ์šฉ๋˜๊ณ , ๋‹ตํ•  ์ˆ˜ ์—†๋Š” ๋™์•ˆ(`stalled`, `stopped`)์—๋Š” `RESOURCE_AUTHORITY_UNAVAILABLE`์ด๋‚˜ `ADMISSION_UNKNOWN`์œผ๋กœ ๊ฑฐ๋ถ€๋˜์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ์žฌ์‹œ์ž‘ํ•œ ๋’ค๋‚˜ ๋ฉˆ์ถ˜ ๋’ค์—๋Š” DevGuard๊ฐ€ ์žฅ์•  ์ „์— ์‹œ์ž‘ํ•œ ๋Œ€์ƒ์„ ํ•ด์ œํ•  ์ˆ˜ ์—†์œผ๋ฏ€๋กœ, ๊ทธ ๋‹จ๊ณ„์—์„œ ํ•ด์ œ๋˜์ง€ ์•Š์€ ๋Œ€์ƒ์€ ์‹คํŒจ๊ฐ€ ์•„๋‹ˆ๋ผ ๋ณด๊ณ  ๋Œ€์ƒ์ž…๋‹ˆ๋‹ค. +`authority`๋Š” fixture authority๊ฐ€ ์ •์ƒ์ด ์•„๋‹ ๋•Œ์˜ ํ”„๋กœ์„ธ์Šค ์ œ์–ด๋ฅผ, runner ๋ชจ๋“œ๋งˆ๋‹ค 15์ดˆ์งœ๋ฆฌ ๋‹ค์„ฏ ๋‹จ๊ณ„๋กœ, ๊ณ„์† ์‹คํ–‰ ์ค‘์ธ ๊ฐ™์€ status ๋Œ€์ƒ์— ๋Œ€ํ•ด ์ธก์ •ํ•ฉ๋‹ˆ๋‹ค. ๋‹จ๊ณ„๋Š” `baseline`, `paused`(reconciler ๋ฉˆ์ถค), `stalled`(๋‹จ๊ณ„ ๋‚ด๋‚ด lock์„ ์žก์Œ), `restarted`, `stopped`(ํ”„๋กœ์„ธ์Šค๋ฅผ ์ฃฝ์ž„)์ž…๋‹ˆ๋‹ค. status๋Š” 1์ดˆ๋งˆ๋‹ค, I/O echo๋Š” 3์ดˆ๋งˆ๋‹ค ์žฌ๊ณ , ์žฅ์•  ์ „์— ์‹œ์ž‘ํ•œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์…‹์„ 3, 7, 11์ดˆ์— ์ข…๋ฃŒํ•˜๋ฉฐ, 1, 5, 9, 13์ดˆ์— ์ƒˆ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์„ ์‹œ๋„ํ•ฉ๋‹ˆ๋‹ค. ๋Œ€์ƒ์€ DevGuard๊ฐ€ ํ—ˆ์šฉํ•œ ๋’ค์— ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. authority๋ฅผ ์žก์•„ ๋‘” ๋™์•ˆ host ํ‘œ๋ณธ์ด ์˜ค๋ž˜๋˜๋ฉด DevGuard๋Š” ํ‘œ๋ณธ์ด 30์ดˆ ๋™์•ˆ ์ •์ƒ์ผ ๋•Œ๊นŒ์ง€ ์ƒˆ ์ž‘์—…์„ ๋ฐ›์ง€ ์•Š์œผ๋ฏ€๋กœ, `stalled` ๋’ค์—๋Š” `RESOURCE_UNAVAILABLE`๋กœ ๊ฑฐ๋ถ€๋œ ๋Œ€์ƒ์„ 90์ดˆ๊นŒ์ง€ 1์ดˆ๋งˆ๋‹ค ๋‹ค์‹œ ์‹œ๋„ํ•˜๊ณ , ๊ฑธ๋ฆฐ ์‹œ๊ฐ„์„ ๋‹จ๊ณ„ ๊ธฐ๋ก์— ๋‚จ๊น๋‹ˆ๋‹ค. ๋‹จ๊ณ„๋งˆ๋‹ค ์œ„์˜ ๊ตฌ๊ฐ„ ๊ทœ์น™์œผ๋กœ ํŒ์ •ํ•˜๋ฉฐ, ์ƒˆ ์‹คํ–‰์€ authority๊ฐ€ ๋‹ตํ•  ์ˆ˜ ์žˆ๋Š” ๋™์•ˆ(`baseline`, `paused`, `restarted`)์—๋Š” ํ—ˆ์šฉ๋˜๊ณ , ๋‹ตํ•  ์ˆ˜ ์—†๋Š” ๋™์•ˆ(`stalled`, `stopped`)์—๋Š” `RESOURCE_AUTHORITY_UNAVAILABLE`์ด๋‚˜ `ADMISSION_UNKNOWN`์œผ๋กœ ๊ฑฐ๋ถ€๋˜์–ด์•ผ ํ•ฉ๋‹ˆ๋‹ค. ์žฌ์‹œ์ž‘ํ•œ ๋’ค๋‚˜ ๋ฉˆ์ถ˜ ๋’ค์—๋Š” DevGuard๊ฐ€ ์žฅ์•  ์ „์— ์‹œ์ž‘ํ•œ ๋Œ€์ƒ์„ ํ•ด์ œํ•  ์ˆ˜ ์—†์œผ๋ฏ€๋กœ, ๊ทธ ๋‹จ๊ณ„์—์„œ ํ•ด์ œ๋˜์ง€ ์•Š์€ ๋Œ€์ƒ์€ ์‹คํŒจ๊ฐ€ ์•„๋‹ˆ๋ผ ๋ณด๊ณ  ๋Œ€์ƒ์ž…๋‹ˆ๋‹ค. @@ -220,7 +220,7 @@ fixture authority๋ฅผ ์“ฐ๋Š” gateway ์‹œํ—˜์€ DevGuard ์‹œํ—˜ ๋ฐ”์ด๋„ˆ๋ฆฌ๊ฐ€ - **๋ฒ”์œ„.** ๊ธฐ๋ณธ๊ฐ’์ด์ž ์„ค์ •์ด ์—†์„ ๋•Œ์˜ ์˜๋ฏธ์ธ ์ฐธ์—ฌ `off`. ๋ชจ๋“  ํ”Œ๋žซํผ๊ณผ runner ๋ชจ๋“œ, pipe์™€ PTY์— ํ•ด๋‹นํ•ฉ๋‹ˆ๋‹ค. ์ด ์‹คํ–‰์€ ์š”์ฒญํ•œ ๋ช…๋ น ์ž์ฒด์ž…๋‹ˆ๋‹ค. pipe ์ž์‹์€ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰๊ณผ ๊ฐ™์€ CodeSpace์˜ reaper ํ•˜๋‚˜๋ฅผ ์“ฐ๊ณ , PTY ์ž์‹์€ reap ์ „ ๊ด€์ฐฐ ์ฐฝ์ด ์—†๋Š” ๊ณ ์ • Codex PTY spawn์„ ์”๋‹ˆ๋‹ค. - **์ด์œ .** ์ž‘์—… ์ง€์‹œ์„œ๋Š” ๊น”๋”ํ•จ๋งŒ์„ ์œ„ํ•ด ์ฝ”๋“œ๋ฅผ ์˜ฎ๊ธฐ์ง€ ์•Š์Šต๋‹ˆ๋‹ค. `off`๋Š” ์ œํ’ˆ์˜ ๊ธฐ๋ณธ๊ฐ’์ด๊ณ , `required`๋Š” ์šด์˜์ž๊ฐ€ ๋ช…์‹œ์ ์œผ๋กœ ๊ณ ๋ฅด๋Š” ๊ฒƒ์ด๋ฉฐ DG-LINUX ์ „๊นŒ์ง€ Linux์—์„œ๋Š” ์ง€์›ํ•˜์ง€ ์•Š์œผ๋ฏ€๋กœ, Linux์—์„œ๋Š” `off`๊ฐ€ ์œ ์ผํ•œ ๊ฒฝ๋กœ์ž…๋‹ˆ๋‹ค. ๊ด€๋ฆฌํ˜• ์‹คํ–‰๋งŒ ๋‚จ๊ธฐ๋ฉด Linux์˜ ๋ชจ๋“  ์‹คํ–‰๊ณผ DevGuard๊ฐ€ ์—†๋Š” ๋ชจ๋“  ํ™˜๊ฒฝ์ด ์‚ฌ๋ผ์ง‘๋‹ˆ๋‹ค. -- **์ค‘๋ณต๋œ ๋ถ€๋ถ„.** spawn ๊ฒฝ๋กœ๋Š” ํ•˜๋‚˜์ด๋ฉฐ ์‹คํ–‰ํ•  ํ”„๋กœ๊ทธ๋žจ๋งŒ ๊ณ ๋ฆ…๋‹ˆ๋‹ค. ์š”์ฒญํ•œ ๋ช…๋ น์ด๊ฑฐ๋‚˜, permit๊ณผ report descriptor๋ฅผ ๋ฐ›๋Š” DevGuard์˜ launch helper์ž…๋‹ˆ๋‹ค([`crates/runner/src/process.rs`](../../crates/runner/src/process.rs)์˜ `How::Direct`์™€ `How::Managed`). reap์€ ์ค‘๋ณต๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์–ด๋Š ์ชฝ์ด๋“  pipe ์ž์‹์€ CodeSpace์˜ reaper ํ•˜๋‚˜๊ฐ€, PTY ์ž์‹์€ ๊ณ ์ • Codex PTY์˜ waiter๊ฐ€ reapํ•ฉ๋‹ˆ๋‹ค. PTY๋Š” ํ˜ธ์ถœ ํ•˜๋‚˜๋งŒ ๋‹ค๋ฆ…๋‹ˆ๋‹ค. ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” ์‹คํ–‰์€ `spawn_inheriting`, ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ [๊ธฐ๋ก๋œ ํŒจ์น˜](upstream-patches.md)์˜ `spawn_observed`๋ฅผ ์”๋‹ˆ๋‹ค. ๊ด€๋ฆฌ ๋Œ€์ƒ ๊ฒฝ๋กœ์—๋Š” ์‹คํ–‰ ํ—ˆ์šฉ, launch, ์ •๋ฆฌ๊ฐ€ ๋”ํ•ด์ง€๋ฉฐ([`crates/runner/src/launch.rs`](../../crates/runner/src/launch.rs)), `off`๋Š” ์ด ๊ฒฝ๋กœ์— ๋“ค์–ด๊ฐ€์ง€ ์•Š์Šต๋‹ˆ๋‹ค. `devguard` feature๊ฐ€ ์—†์œผ๋ฉด ๋นŒ๋“œ์— DevGuard ํŒจํ‚ค์ง€๊ฐ€ ์—†๊ณ , ์žˆ์œผ๋ฉด DevGuard์˜ client์™€ contract crate๋งŒ ๋“ค์–ด๊ฐ€๋ฉฐ, `dependencies` ๋‹จ๊ณ„๊ฐ€ ์ด๋ฅผ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. +- **์ค‘๋ณต๋œ ๋ถ€๋ถ„.** spawn ๊ฒฝ๋กœ๋Š” ํ•˜๋‚˜์ด๋ฉฐ ์‹คํ–‰ํ•  ํ”„๋กœ๊ทธ๋žจ๋งŒ ๊ณ ๋ฆ…๋‹ˆ๋‹ค. ์š”์ฒญํ•œ ๋ช…๋ น์ด๊ฑฐ๋‚˜, permit๊ณผ report descriptor๋ฅผ ๋ฐ›๋Š” DevGuard์˜ launch helper์ž…๋‹ˆ๋‹ค([`crates/runner/src/process.rs`](../../crates/runner/src/process.rs)์˜ `How::Direct`์™€ `How::Managed`). reap์€ ์ค‘๋ณต๋˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์–ด๋Š ์ชฝ์ด๋“  pipe ์ž์‹์€ CodeSpace์˜ reaper ํ•˜๋‚˜๊ฐ€, PTY ์ž์‹์€ ๊ณ ์ • Codex PTY์˜ waiter๊ฐ€ reapํ•ฉ๋‹ˆ๋‹ค. PTY๋Š” ํ˜ธ์ถœ ํ•˜๋‚˜๋งŒ ๋‹ค๋ฆ…๋‹ˆ๋‹ค. ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” ์‹คํ–‰์€ `spawn_inheriting`, ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ [๊ธฐ๋ก๋œ ํŒจ์น˜](upstream-patches.md)์˜ `spawn_observed`๋ฅผ ์”๋‹ˆ๋‹ค. ๊ด€๋ฆฌ ๋Œ€์ƒ PTY ์ž์‹์€ launch descriptor๋„ ๋ฌผ๋ ค๋ฐ›์œผ๋ฏ€๋กœ ๊ณ ์ • Codex PTY๋Š” descriptor๋ฅผ ์œ ์ง€ํ•˜๋Š” ๊ฒฝ๋กœ๋กœ ๋„์šฐ๊ณ , ์ด ๊ฒฝ๋กœ๋Š” ์‹ ํ˜ธ๋กœ ๋๋‚œ ๊ฒƒ์„ 128์— ์‹ ํ˜ธ ๋ฒˆํ˜ธ๋ฅผ ๋”ํ•œ exit code๋กœ ๋ณด๊ณ ํ•ฉ๋‹ˆ๋‹ค. ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” PTY๋Š” 1๋กœ ๋ณด๊ณ ํ•ฉ๋‹ˆ๋‹ค([ํ”„๋กœ์„ธ์Šค ์ƒํƒœ](agent-integration.md)). ๊ด€๋ฆฌ ๋Œ€์ƒ ๊ฒฝ๋กœ์—๋Š” ์‹คํ–‰ ํ—ˆ์šฉ, launch, ์ •๋ฆฌ๊ฐ€ ๋”ํ•ด์ง€๋ฉฐ([`crates/runner/src/launch.rs`](../../crates/runner/src/launch.rs)), `off`๋Š” ์ด ๊ฒฝ๋กœ์— ๋“ค์–ด๊ฐ€์ง€ ์•Š์Šต๋‹ˆ๋‹ค. `devguard` feature๊ฐ€ ์—†์œผ๋ฉด ๋นŒ๋“œ์— DevGuard ํŒจํ‚ค์ง€๊ฐ€ ์—†๊ณ , ์žˆ์œผ๋ฉด DevGuard์˜ client์™€ contract crate๋งŒ ๋“ค์–ด๊ฐ€๋ฉฐ, `dependencies` ๋‹จ๊ณ„๊ฐ€ ์ด๋ฅผ ํ™•์ธํ•ฉ๋‹ˆ๋‹ค. - **`off`๊ฐ€ ์ œ๊ณตํ•˜์ง€ ์•Š๋Š” ๊ฒƒ.** DevGuard์˜ ์‹คํ–‰ ํ—ˆ์šฉ, ์˜ˆ์•ฝ, ํšŒ๊ณ„๊ฐ€ ์—†๊ณ , scope ๋‹จ์†์ด ์—†์œผ๋ฉฐ, ๊ด€๋ฆฌํ•˜์ง€ ์•Š๋Š” PTY root๋ฅผ reap ์ „์— ๊ด€์ฐฐํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. Linux ๋ช…๋ น sandbox๋Š” `off` ์‹คํ–‰๋งŒ ์“ธ ์ˆ˜ ์žˆ๊ณ , sandbox๊ฐ€ ์ผœ์ง„ ๊ณณ์—์„œ ๊ด€๋ฆฌ ๋Œ€์ƒ ์‹คํ–‰์€ ๊ฑฐ๋ถ€๋ฉ๋‹ˆ๋‹ค. - **์žฌ๊ฒ€ํ†  ์‹œ์ .** DG-LINUX๊ฐ€ Linux ๊ฐ•์ œ ๋ณดํ˜ธ๋ฅผ qualificationํ•  ๋•Œ, Codex pin ์ด๋™์œผ๋กœ PTY spawn์ด ๋ฐ”๋€” ๋•Œ, `required`๊ฐ€ ๊ธฐ๋ณธ๊ฐ’์ด ๋  ๋•Œ์ž…๋‹ˆ๋‹ค. - **์ œ๊ฑฐ ๊ธฐ์ค€.** ๋…๋ฆฝ์ ์ธ ์ œํ’ˆ์ƒ์˜ ์ด์œ ๊ฐ€ ์žˆ๊ณ , ์ด ๊ฒฝ๋กœ ์—†์ด๋„ ์ง€์›ํ•˜๋Š” ๋ชจ๋“  ๋ชจ๋“œ์˜ ๋™์ž‘์ด ์œ ์ง€๋œ๋‹ค๋Š” ํšŒ๊ท€ ์ฆ๊ฑฐ๊ฐ€ ์žˆ์„ ๋•Œ๋งŒ ์ œ๊ฑฐํ•ฉ๋‹ˆ๋‹ค. diff --git a/docs/translations.json b/docs/translations.json index 97812a7..c46fc08 100644 --- a/docs/translations.json +++ b/docs/translations.json @@ -130,8 +130,8 @@ "ํšŒ๊ท€์™€-๋™๋“ฑ์„ฑ", "ํ›„๋ณด-์กฐํ•ฉ" ], - "source_sha256": "018ad1ba83f5f63c2442d2d76b818ecee88742e5698404f3ef1d6c40d67fe0ac", - "translation_sha256": "1cdf7e966d26809d9a35689a12489a9444322c46d497d906e795e0b842175076" + "source_sha256": "99d92fa6d0a302b40c70b892f5af601e662e866f4ff99cd612220577d6149128", + "translation_sha256": "3fc102df0febc47abe134718152cddd532cdc1245c997a6570f246fc3c0c6510" }, { "id": "overview", @@ -204,8 +204,8 @@ "์žฌ์‹œ๋„์™€-๋ณต๊ตฌ", "ํŒŒ์ผ-์ฝ๊ธฐ์™€-ํŒจ์น˜" ], - "source_sha256": "5c89ffabca2002b3473a5c45dc52ea6bae522da36ca4cfbf8db657bf997ba96f", - "translation_sha256": "0d3c33de0a8bbdaffc87970ce8ab243871df9d284f9cb1d4de5b953dc6dd6e6e" + "source_sha256": "eb38ede0d2c1f2457e238c92ee554c7dc292ba50841835f0c658431f6c9a6f33", + "translation_sha256": "a71c333f14a70d9848e14fe1633e34de3d9616fca1ab5e432d0ed1387aa3c218" }, { "id": "operations", diff --git a/scripts/qualify-devguard.py b/scripts/qualify-devguard.py index 9b6631b..397aa82 100644 --- a/scripts/qualify-devguard.py +++ b/scripts/qualify-devguard.py @@ -467,8 +467,10 @@ def scenario_registration(bench, gateway): def scenario_exits(bench, gateway, tty): checks, detail = [], {} - # The pinned Codex PTY reports a signal death as exit code 1 (docs/agent-integration.md). - death = {'termination': 'exited', 'exit_code': 1} if tty else {'termination': 'signaled', 'signal': 15} + # A governed PTY child inherits its launch descriptors, so the pinned Codex PTY spawns it on + # its descriptor-keeping path, which reports a signal death as exit code 128 + the signal + # (docs/agent-integration.md). + death = {'termination': 'exited', 'exit_code': 128 + 15} if tty else {'termination': 'signaled', 'signal': 15} for label, command, expected in (('exit 0', ['true'], {'termination': 'exited', 'exit_code': 0}), ('exit 3', ['sh', '-c', 'exit 3'], {'termination': 'exited', 'exit_code': 3}), ('death by SIGTERM', ['sh', '-c', 'kill -TERM $$'], death)): @@ -719,19 +721,21 @@ def matrix_workspaces(work, capacity): 'hog': (work / 'ws/hog', host.required(**hog))} -def settled_checks(bench, label): - """After a mode: nothing of the consumer is charged, and its instances are not active.""" - deadline = time.monotonic() + 60 +def settled_checks(bench, label, limit=60.0, period=1.0): + """After a mode: nothing of the consumer is charged, and its instances are not active. DevGuard + retires an instance once it sees that its owner is gone, which can come after the gateway has + exited, so both are awaited for up to `limit` seconds.""" + deadline = time.monotonic() + limit while True: charged = host.journal_charged(bench.database) mine = charged['attempts'].get(bench.authority['consumer'], 0) - if not mine or time.monotonic() >= deadline: + instances = host.journal_instances(bench.authority['consumer'], bench.database) + active = [entry for entry in instances if entry['state'] == 'active'] + if (not mine and not active) or time.monotonic() >= deadline: break - time.sleep(1) - instances = host.journal_instances(bench.authority['consumer'], bench.database) + time.sleep(period) return [check(f'{label}: nothing of the consumer stays charged', mine == 0, charged=charged), - check(f'{label}: no instance stays active', not any(entry['state'] == 'active' for entry in instances), - instances=instances)] + check(f'{label}: no instance stays active', not active, instances=instances)] def installed_authority(args): @@ -778,7 +782,8 @@ def matrix(args, live, evidence): run_scenario(results, evidence, mode, None, 'runner-socket', scenario_runner_socket, bench) entry = run_scenario(results, evidence, mode, None, 'settled', lambda: (settled_checks(bench, mode), {})) if entry['verdict'] != 'pass': - print('qualify: work stayed charged; the matrix stops here', flush=True) + print('qualify: work stayed charged or an instance stayed active; the matrix stops here', + flush=True) break finally: if bench is not None: @@ -1534,6 +1539,29 @@ def raw_hashes(directory): ('stalled', 'hold', 'refused', True), ('restarted', 'restart', 'admitted', False), ('stopped', 'kill', 'refused', False)) REFUSED_UNREACHABLE = ('RESOURCE_AUTHORITY_UNAVAILABLE', 'ADMISSION_UNKNOWN') +# A host sample that goes stale while the authority is held makes DevGuard close new work until +# its samples have been clear for 30 seconds, so a phase after `stalled` waits that long, and +# some margin, for its termination targets to be admitted. +READMIT_LIMIT_S = 90.0 + + +def start_targets(gateway, count, limit=READMIT_LIMIT_S, period=1.0): + """A phase's termination targets, alternating pipe and PTY. A target refused for capacity or + pressure (`RESOURCE_UNAVAILABLE`) is tried again every `period` seconds until it is admitted + or `limit` seconds have passed. Returns the targets, the refusals met, and how long it took.""" + began = time.monotonic() + pool, refusals = [], 0 + for n in range(count): + tty = bool(n % 2) + while True: + entry = gateway.exec(f'term-{n}', ['/bin/sleep', '600'], tty=tty) + if (entry['outcome'] == 'started' or entry.get('code') != 'RESOURCE_UNAVAILABLE' + or time.monotonic() - began >= limit): + break + refusals += 1 + time.sleep(period) + pool.append(dict(entry, tty=tty)) + return pool, refusals, round(time.monotonic() - began, 3) def authority_workspaces(work): @@ -1566,15 +1594,14 @@ def phase_verdict(record, rows): def authority_phase(bench, gateway, evidence, mode, targets, echo, phase, fault, expectation, release_expected): - """One phase: its termination targets started before the fault, the fault applied, the - schedules run for PHASE_S seconds, then the fault lifted where it can be. The samples have - their own writer, closed before their files are hashed.""" + """One phase: its termination targets started before the fault, once DevGuard admits them, + the fault applied, the schedules run for PHASE_S seconds, then the fault lifted where it can + be. The samples have their own writer, closed before their files are hashed.""" raw = evidence.file(f'{mode}/authority/{phase}/x').parent samples = host.Writer() schedules, watcher, invalid = {}, None, [] try: - pool = [dict(gateway.exec(f'term-{n}', ['/bin/sleep', '600'], tty=bool(n % 2)), tty=bool(n % 2)) - for n in range(len(PHASE_TERMINATIONS))] + pool, refusals, admitted_s = start_targets(gateway, len(PHASE_TERMINATIONS)) prepared = [entry['process_id'] for entry in pool if entry['outcome'] == 'started'] if len(prepared) < len(pool): invalid.append('a termination target could not be started before the fault') @@ -1628,6 +1655,7 @@ def authority_phase(bench, gateway, evidence, mode, targets, echo, phase, fault, invalid.append(f"raw evidence could not be written: {written['errors'][:3]}") record = {'mode': mode, 'phase': phase, 'fault': fault, 'applied': applied, 'started_unix': started_unix, 'seconds': seconds, 'release_expected': release_expected, 'expectation': expectation, + 'termination_targets': {'refusals_before_admitted': refusals, 'admitted_s': admitted_s}, 'invalid': invalid, 'checks': phase_checks(expectation, rows['refusals']), 'read_errors': watcher.read_errors, 'evidence_writer': written, 'raw': raw_hashes(raw)} record['verdict'], record['reasons'], record['metrics'] = phase_verdict(record, rows) diff --git a/scripts/tests/test_qualify_devguard.py b/scripts/tests/test_qualify_devguard.py index 5287181..297d664 100644 --- a/scripts/tests/test_qualify_devguard.py +++ b/scripts/tests/test_qualify_devguard.py @@ -13,7 +13,9 @@ import tempfile import textwrap import time +from types import SimpleNamespace import unittest +from unittest import mock SCRIPTS = Path(__file__).resolve().parents[1] sys.path.insert(0, str(SCRIPTS)) @@ -544,6 +546,44 @@ def test_phase_expectations(self): self.assertFalse(qualify.phase_checks('refused', refused(4))[0]['passed']) self.assertFalse(qualify.phase_checks('refused', [{'slot': 0, 'outcome': 'missed_slot'}])[0]['passed']) + def test_targets_wait_until_devguard_admits_them(self): + class Gateway: + def __init__(self, refusals, code='RESOURCE_UNAVAILABLE'): + self.refusals, self.code, self.calls = refusals, code, [] + + def exec(self, workspace, command, tty=False): + self.calls.append(workspace) + if self.refusals: + self.refusals -= 1 + return {'outcome': 'refused', 'code': self.code} + return {'outcome': 'started', 'process_id': f'proc-{len(self.calls)}'} + + gateway = Gateway(2) + pool, refusals, _ = qualify.start_targets(gateway, 3, period=0) + self.assertEqual([(entry['outcome'], entry['tty']) for entry in pool], + [('started', False), ('started', True), ('started', False)]) + self.assertEqual((refusals, gateway.calls), (2, ['term-0'] * 3 + ['term-1', 'term-2'])) + # Only a refusal for capacity or pressure is awaited, and only up to the limit. + pool, refusals, _ = qualify.start_targets(Gateway(1, 'RESOURCE_AUTHORITY_UNAVAILABLE'), 1, period=0) + self.assertEqual((pool[0]['outcome'], refusals), ('refused', 0)) + pool, refusals, _ = qualify.start_targets(Gateway(10 ** 6), 1, limit=0.05, period=0.01) + self.assertEqual(pool[0]['outcome'], 'refused') + self.assertGreater(refusals, 0) + + def test_settled_awaits_charges_and_instances(self): + bench = SimpleNamespace(database=None, authority={'consumer': 'codespace'}) + charged = iter([{'attempts': {'codespace': 1}, 'leases': 0}] + [{'attempts': {}, 'leases': 0}] * 3) + instances = iter([[{'state': 'active'}]] * 2 + [[{'state': 'retired'}]] * 2) + with mock.patch.object(host, 'journal_charged', lambda database: next(charged)), \ + mock.patch.object(host, 'journal_instances', lambda consumer, database: next(instances)): + checks = qualify.settled_checks(bench, 'uds', period=0) + self.assertEqual([item['passed'] for item in checks], [True, True], checks) + # An instance still active once the limit has passed fails, though nothing is charged. + with mock.patch.object(host, 'journal_charged', lambda database: {'attempts': {}, 'leases': 0}), \ + mock.patch.object(host, 'journal_instances', lambda consumer, database: [{'state': 'active'}]): + checks = qualify.settled_checks(bench, 'uds', limit=0.05, period=0.01) + self.assertEqual([item['passed'] for item in checks], [True, False], checks) + def test_repetition_outcomes(self): report = {'service_before': {'runs_release': True, 'service_pid': 7}, 'service_after': {'runs_release': True, 'service_pid': 7}, 'rehearsal': False}