FEATURE-ci-darwin-header-deadline: Prepare native archive deadline diagnostic checkpoint

Add temporary bounded compile-time traces for the actual watchdog, timer completion, external dispatch and socket close path. Manually run only macOS against the exact failing c0 selected archive plus an explicitly hashed diagnostic overlay. Reuse strict artifact verification and extraction, keep read-only permissions, and preserve push/PR matrix commands and original test budgets/assertions. Keep the task In Progress for native evidence and later TDD repair/removal. Enabled/disabled syntax, trace contract, workflow syntax and changed-file cpplint passed; actual selected archive overlay and extracted-source syntax preflight passed. No full baseline, repair, push or merge.

Author: etr

.github/workflows/v3-package-validation.yml

@@ -9,6 +9,7 @@ permissions:
   contents: read
 jobs:
   linux:
+    if: github.event_name != 'workflow_dispatch'
     runs-on: ubuntu-24.04
     timeout-minutes: 90
     steps:
@@ -44,6 +45,9 @@ jobs:
           retention-days: 14
           if-no-files-found: error
   macos:
+    permissions:
+      contents: read
+      actions: read
     runs-on: macos-15
     timeout-minutes: 90
     steps:
@@ -67,12 +71,125 @@ jobs:
       - name: Bootstrap
         run: ./bootstrap
       - name: Installed matrix
+        if: github.event_name != 'workflow_dispatch'
         run: |
           prefix=()
           if ! python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",443)); s.close()'; then
             prefix=(sudo env "PATH=$PATH" GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=safe.directory "GIT_CONFIG_VALUE_0=$PWD")
           fi
           "${prefix[@]}" python3 scripts/check-v3-installed-consumer.py --output "$RUNNER_TEMP/package-evidence/matrix" --provider-prefix "$RUNNER_TEMP/package-evidence/prerequisites/provider" --pkg-config-dir "$(brew --prefix)/share/pkgconfig" --client-python "$RUNNER_TEMP/package-evidence/clients/bin/python" --node node --compiler "$V3_CXX" --cc "$V3_CC" --toolchain-prefix "$V3_PACKAGE_TOOLCHAIN" --jobs 2
+      # Branch-only checkpoint: remove before merging to v3. This is not matrix proof.
+      - name: Diagnose installed shared TLS-off external HTTP1 on Darwin
+        if: github.event_name == 'workflow_dispatch'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          python3 - <<'PY_DIAGNOSTIC'
+          import hashlib, importlib.util, json, os, platform, shlex, shutil, subprocess, sys
+          from pathlib import Path
+          checkout = Path.cwd().resolve()
+          sys.path.insert(0, str(checkout / 'scripts/lib'))
+          import package_matrix as matrix
+          from package_commands import Commands, runtime_environment
+          from package_integration import compile_probe
+          import release_artifact as identity
+          from release_checks import extract_archive
+          from release_ci import verify_download
+          spec = importlib.util.spec_from_file_location('audit', checkout / 'scripts/audit-v3-installed-package.py')
+          audit = importlib.util.module_from_spec(spec)
+          spec.loader.exec_module(audit)
+          assert platform.system() == 'Darwin'
+          output = Path(os.environ['RUNNER_TEMP']) / 'package-evidence/header-deadline-diagnostic'
+          output.mkdir()
+          build, prefix, integration = [output / name for name in ('build', 'prefix', 'integration-shared')]
+          for directory in (build, prefix, integration):
+              directory.mkdir()
+          command = Commands(output)
+          provider = output.parent / 'prerequisites/provider'
+          compiler, cc = os.environ['V3_CXX'], os.environ['V3_CC']
+          toolchain = Path(os.environ['V3_PACKAGE_TOOLCHAIN'])
+          env = audit.clean_environment()
+          for key in ('GH_TOKEN', 'GITHUB_TOKEN', 'V3_RELEASE_IDENTITY'):
+              env.pop(key, None)
+          env.update(CXX=compiler, CC=cc, CXXFLAGS='-O0 -g',
+                     CPPFLAGS='-DHTTPSERVER_DIAG_HEADER_DEADLINE',
+                     LDFLAGS='-Wl,-rpath,' + str(provider / 'lib'))
+          receipt = dict(kind='diagnostic-only; not complete package validation', tls='no', linkage='shared',
+                         revision=command(['git', 'rev-parse', 'HEAD'], checkout, env).strip(),
+                         compiler=command([compiler, '--version'], checkout, env),
+                         provider_header_sha256=hashlib.sha256((provider / 'include/openssl/opensslv.h').read_bytes()).hexdigest())
+          try:
+              expected = dict(repository=os.environ['GITHUB_REPOSITORY'], run_id='39969790346', run_attempt='1',
+                              workflow_head='c0cfdabc8040273d60c610269a93d1c687ed8517', artifact_id='12508482722')
+              endpoint = 'repos/' + expected['repository'] + '/actions/artifacts/' + expected['artifact_id']
+              metadata = json.loads(subprocess.check_output(['gh', 'api', endpoint], text=True))
+              run = json.loads(subprocess.check_output(['gh', 'api', 'repos/' + expected['repository'] +
+                  '/actions/runs/' + expected['run_id'] + '/attempts/' + expected['run_attempt']], text=True))
+              (output / 'selected-artifact-metadata.json').write_text(json.dumps(metadata, indent=2) + '\n')
+              (output / 'selected-run-metadata.json').write_text(json.dumps(run, indent=2) + '\n')
+              downloaded = output / 'selected-archive.zip'
+              with downloaded.open('xb') as destination:
+                  subprocess.run(['gh', 'api', endpoint + '/zip'], stdout=destination, check=True)
+              zip_digest = 'sha256:a08eb79186b13cf12ac8c639f424e13a7de1ee09df5547aab2b4200a951a5f05'
+              verify_download(downloaded, metadata, run, expected, zip_digest, output / 'selected')
+              selected = json.loads((output / 'selected/selected.json').read_text())
+              archive = output / 'selected/libhttpserver-3.0.0.tar.gz'
+              assert selected['commit'] == selected['source']['head'] == expected['workflow_head']
+              assert selected['archive'] == archive.name and archive.stat().st_size == 3053890
+              assert identity.sha256(archive) == selected['sha256'] == '43124552f20bd00fceddc714856d605ca9caf017a849a991c34f16f437fcab83'
+              source = extract_archive(archive, output / 'source')
+              originals = identity.source_files(source)
+              assert originals and all(selected['files_sha256'].get(name) == digest for name, digest in originals.items())
+              (output / 'original-source-hashes.json').write_text(json.dumps(originals, indent=2) + '\n')
+              overlay = {}
+              names = ['src/detail/' + name + '.cpp' for name in ('connection_engine', 'connection_engine_websocket',
+                  'io_external_adapter', 'io_operation', 'io_poll_backend', 'io_poll_sys', 'io_timer_sweep')]
+              names += ['test/integ/external_http1_e2e_test.cpp', 'src/httpserver/detail/deadline_trace.hpp']
+              for name in names:
+                  original = originals.get(name)
+                  if name.endswith('/deadline_trace.hpp'):
+                      assert original is None and name not in selected['source']['files_sha256']
+                  else:
+                      assert original == selected['source']['files_sha256'][name]
+                  overlay[name] = dict(original_sha256=original, diagnostic_sha256=identity.sha256(checkout / name))
+                  shutil.copyfile(checkout / name, source / name)
+              modified = identity.source_files(source)
+              assert set(modified) - set(originals) == {'src/httpserver/detail/deadline_trace.hpp'}
+              assert {name for name in originals if modified[name] != originals[name]} == set(names[:-1])
+              (output / 'diagnostic-source-hashes.json').write_text(json.dumps(modified, indent=2) + '\n')
+              receipt['source_archive'] = dict(**expected, artifact_zip_sha256=zip_digest,
+                  archive_sha256=selected['sha256'], source_root=str(source), overlay=overlay,
+                  provenance='verified original archive plus explicit diagnostic overlay; not immutable release proof')
+              command(['bash', source / 'configure', '--enable-v3-tls=no', '--disable-examples', '--prefix=' + str(prefix)], build, env)
+              command(['make', '-j2'], build, env)
+              command(['make', 'install'], build, env)
+              runtime = runtime_environment({key: value for key, value in audit.clean_environment().items()
+                  if key not in ('GH_TOKEN', 'GITHUB_TOKEN', 'V3_RELEASE_IDENTITY')}, prefix, provider, [], toolchain)
+              runtime.update(PKG_CONFIG_PATH='', PKG_CONFIG_LIBDIR=str(prefix / 'lib/pkgconfig'))
+              os.environ.update({key: runtime[key] for key in ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'V3_PACKAGE_TOOLCHAIN')})
+              cflags = shlex.split(command(['pkg-config', '--cflags', 'libhttpserver'], integration, runtime))
+              libraries = shlex.split(command(['pkg-config', '--libs', 'libhttpserver'], integration, runtime))
+              libraries += ['-pthread', '-Wl,-rpath,' + str(prefix / 'lib'), '-Wl,-rpath,' + str(provider / 'lib')]
+              flags = cflags + ['-DHTTPSERVER_COMPILATION', '-DHTTPSERVER_DIAG_HEADER_DEADLINE',
+                                '-I' + str(source / 'src'), '-I' + str(source / 'test')]
+              definition = matrix.load()['suites']['external_http1_e2e']
+              units = [source / 'test' / name for name in definition['sources']]
+              executable = integration / 'external_http1_e2e'
+              receipt['compile'] = compile_probe(compiler, units, flags, libraries, executable, command, runtime, source, build, prefix, True)
+              graph = audit.dependency_graph([str(executable)])
+              matrix.check_linkage(graph, 'shared', prefix)
+              receipt['dependencies'] = graph
+              result = command([str(executable)], source / 'test', runtime, timeout=180, allow_failure=True)
+              print(result, end='', flush=True)
+              receipt['exit'] = command.records[-1]['exit']
+              receipt['suite'] = matrix.suite_result(result, receipt['exit'])
+              receipt['status'] = 'diagnostic suite passed; full matrix not run'
+          except Exception as error:
+              receipt.update(status='diagnostic failed', error=str(error))
+              raise
+          finally:
+              (output / 'diagnostic.json').write_text(json.dumps(receipt, indent=2) + '\n')
+          PY_DIAGNOSTIC
       - name: Make privileged proof readable by artifact upload
         if: always()
         run: |
@@ -89,6 +206,7 @@ jobs:
           retention-days: 14
           if-no-files-found: error
   mingw64:
+    if: github.event_name != 'workflow_dispatch'
     runs-on: windows-2022
     timeout-minutes: 90
     defaults:
@@ -148,6 +266,7 @@ jobs:
           retention-days: 14
           if-no-files-found: error
   freebsd:
+    if: github.event_name != 'workflow_dispatch'
     runs-on: ubuntu-24.04
     timeout-minutes: 180
     steps:

specs/tasks/FEATURE-ci-darwin-header-deadline.md

@@ -0,0 +1,112 @@
+# FEATURE-ci-darwin-header-deadline: Diagnose native external header deadline
+
+Status: In Progress — diagnostic checkpoint only; awaiting native Darwin evidence.
+
+Base: `v3` at `c0cfdabc8040273d60c610269a93d1c687ed8517`.
+
+Plan: `.groundwork-plans/FEATURE-ci-darwin-header-deadline-plan.md` in the primary checkout.
+Journal: `.debug/ci-darwin-native-69af4dxe/journal.md` in the primary checkout.
+Prior closed diagnostic: `.debug/ci-darwin-header-deadline-ybz8bi7l/journal.md`.
+
+## Goal and evidence
+
+Determine why the installed shared TLS-off Darwin `external_http1_e2e`
+`published_header_deadline_drives_empty_dispatch` case fails to observe peer close,
+then repair only the proven cause. User authorized focused native diagnostics on
+2026-10-23. Preserve the 300 ms header deadline, 10 s host failure deadline and all
+existing assertions.
+
+The current base reproduces the failure in archive run `39969790346`, Darwin job
+`120469466623`, artifact `12509855171`: four tests, 52 checks, two failures,
+10,006.1 ms for the deadline case. Assertions 217 (`receive_close`) and 221
+(`!failed`) fail; assertion 219 (`empty_dispatches > 0`) passes. Other three cases
+and the earlier native HTTP1 suite pass. The underlying cause remains unknown.
+
+New native comparison before publishing the checkpoint: current c0 package job
+`120468733495` passes the entire TLS-off lane, including this external deadline
+case, then fails later in TLS-on public TLS with Python
+`SSLCertVerificationError` (CA certificate missing key usage). Archive job
+`120469466623` fails the header deadline at the same source revision, selected
+LLVM toolchain and `-O0 -g`. Checkout versus extracted archive/execution context
+is now an observed distinction; it is not an established cause. The CA issue is
+queued separately and is outside this task.
+
+## Action items
+
+- [x] Preserve and read prior/current evidence without repeating the closed Linux probe.
+- [x] Create isolated task branch/worktree from the exact base.
+- [x] Add compile-time optional, bounded traces to actual watchdog, timer,
+  dispatch, registration retirement and POSIX native close paths.
+- [x] Prepare a branch-only manual Darwin diagnostic in the existing
+  `v3-package-validation.yml`; preserve push/PR matrix commands and skip other
+  jobs explicitly on manual dispatch.
+- [x] Check enabled/disabled trace compilation, bounded output, errno preservation,
+  ordinary-build argument elision, workflow syntax and changed-file lint.
+- [ ] Execute the native diagnostic and retain actual installed-library provenance.
+- [ ] Establish cause from native evidence, then write and observe a failing
+  regression before applying a narrow repair.
+- [ ] Remove all temporary trace code/header and workflow edits before final merge.
+- [ ] Run task-relevant final checks and Groundwork validation/finalization.
+
+## Checkpoint scope and checks
+
+`HTTPSERVER_DIAG_HEADER_DEADLINE` enables temporary stderr records, capped at 64
+per call site. Correlation uses connection IDs, operation pointers/sequences,
+socket descriptors and registration generations. Time points are converted with
+`duration_cast<nanoseconds>`; header durations are explicit milliseconds. No
+request bytes, credentials or peer content are logged. Ordinary builds erase
+trace argument evaluation.
+
+Manual dispatch runs macOS only, preserving its selected compiler/runtime,
+provider setup and artifact upload. It builds and installs TLS-off libraries at
+`-O0 -g`, then uses existing `Commands`, `runtime_environment`, `compile_probe`,
+dependency inspection, shared-linkage checks and suite receipt parsing to run
+the existing four-case external HTTP1 integration suite against the installed
+shared library built from the actual failing c0 selected archive. The macOS job
+uses only `contents: read` and `actions: read` with the existing GitHub token to
+download artifact `12508482722` from run `39969790346` attempt 1. Existing
+`verify_download` validates artifact/run/repository/source binding and ZIP digest
+`a08eb79186b13cf12ac8c639f424e13a7de1ee09df5547aab2b4200a951a5f05`.
+The archive is exactly 3,053,890 bytes with SHA-256
+`43124552f20bd00fceddc714856d605ca9caf017a849a991c34f16f437fcab83`.
+Existing strict `extract_archive` extracts it without a checkout `.git` directory.
+All original packaged file hashes must match the selected descriptor. Before
+copying the seven traced private implementations and existing integration test,
+each original hash must also match the c0 source snapshot. The private trace
+header is the sole added file; every other source remains byte-identical to the
+archive. Original/modified source manifests and per-file overlay hashes are
+retained. The original archive is preserved separately; the overlaid source is
+explicitly a diagnostic modified snapshot, not immutable release proof, and does
+not set `V3_RELEASE_IDENTITY`. Build and test source paths/cwd use this extracted
+tree. No build command receives the download token.
+
+Compile/dependency hashes, commands, runtime selection and the failing suite log
+are retained in `package-evidence/header-deadline-diagnostic`. This partial run is
+explicitly diagnostic evidence and does not satisfy the package matrix gate.
+
+Local checks passed: eight affected translation units with traces disabled and
+enabled (`g++ -fsyntax-only`, format errors enabled), trace contract checks,
+workflow YAML/Python parsing and shell syntax, and cpplint on all changed C++
+files. The first syntax invocation lacked generated `config.h`; bootstrap plus
+an out-of-tree TLS-off configure corrected that execution setup, and both
+configured invocations passed. No full baseline or old Linux publication/wait
+probe was run. The follow-up archive-wiring preflight uses the actual downloaded
+selected artifact locally: exact ZIP/tar and original hashes pass, the literal
+workflow extraction/overlay block passes, all eight original overlay files match
+the exact base, and the private trace header is present. All eight affected
+translation units also pass diagnostic-enabled syntax/format checks using only
+the extracted source/include paths. Workflow syntax passes again; only workflow provenance and this task record change in the follow-up.
+Darwin execution and the original required gate remain incomplete.
+
+The caller reviews/publishes only this task branch and dispatches the existing
+workflow, then resumes this same worktree with actual native evidence. No root
+cause, behavior repair, validation completion or merge is claimed here. Tracing
+can perturb scheduling, so a passing diagnostic alone would not prove a fix.
+
+## Scope boundaries
+
+The older Linux parity uncertainty remains closed. This task does not authorize
+parity repairs, broad platform reruns, timeout changes, assertion relaxation,
+dependency/policy changes or a new shared verification framework. Preserve the
+unrelated bauth worktree. Nonlocal final platform checks remain assigned to CI
+and the v3 PR under the repository working agreement.

src/detail/connection_engine.cpp

@@ -41,6 +41,8 @@
 #include <httpserver/detail/diagnostics_access.hpp>
 #include <httpserver/http/outcome.hpp>
 
+#include <httpserver/detail/deadline_trace.hpp>
+
 namespace httpserver {
 
 namespace detail {
@@ -715,6 +717,9 @@ bool connection_engine::watchdog_due(
     std::chrono::steady_clock::time_point deadline) {
     std::lock_guard<std::mutex> lock(mu_);
     const watchdog_plan plan = plan_watchdog_locked();
+    V3_DEADLINE_TRACE("due conn=%llu planned_ns=%lld current_ns=%lld now_ns=%lld exit=%d defer=%d equal=%d\n",
+        static_cast<deadline_trace_id>(id_), deadline_trace_ns(deadline), deadline_trace_ns(plan.deadline),
+        deadline_trace_ns(std::chrono::steady_clock::now()), plan.exit, plan.defer, plan.deadline == deadline);
     return !plan.exit && !plan.defer && plan.deadline == deadline
         && std::chrono::steady_clock::now() >= deadline;
 }
@@ -740,6 +745,8 @@ void connection_engine::note_transport_activity(bool written) {
 }
 
 void connection_engine::enforce_timeout(std::chrono::steady_clock::time_point deadline) noexcept {
+    V3_DEADLINE_TRACE("enforce conn=%llu deadline_ns=%lld now_ns=%lld\n",
+        static_cast<deadline_trace_id>(id_), deadline_trace_ns(deadline), deadline_trace_ns(std::chrono::steady_clock::now()));
     bool global_expiry = false, close_expiry = false;
     {
         std::lock_guard<std::mutex> lock(mu_);
@@ -776,13 +783,19 @@ task<void> connection_engine::watchdog_loop(
             std::lock_guard<std::mutex> lock(self->mu_);
             plan = self->plan_watchdog_locked();
         }
+        V3_DEADLINE_TRACE("plan conn=%llu deadline_ns=%lld now_ns=%lld exit=%d defer=%d\n",
+            static_cast<deadline_trace_id>(self->id_), deadline_trace_ns(plan.deadline),
+            deadline_trace_ns(std::chrono::steady_clock::now()), plan.exit, plan.defer);
         if (plan.exit) co_return;
         const auto deadline = plan.defer
             ? std::chrono::steady_clock::now() + k_watchdog_tick
             : plan.deadline;
         timer_operation op(self->owner_, self->id_, deadline);
         if (!self->arm_watchdog(op, plan)) continue;
         const io_result r = co_await std::move(op);
+        V3_DEADLINE_TRACE("watchdog_resume conn=%llu seq=%llu result=%d now_ns=%lld\n",
+            static_cast<deadline_trace_id>(self->id_), static_cast<deadline_trace_id>(op.sequence()),
+            static_cast<int>(r.code), deadline_trace_ns(std::chrono::steady_clock::now()));
         {
             std::lock_guard<std::mutex> lock(self->mu_);
             self->pending_timer_.reset();