Skip to content

Commit e6d1da5

Browse files
committed
FEATURE-ci-audit-fixtures-and-evidence: preserve failed inspections and portable audit fixtures
Use PE fixture artifacts and native fixture roots while preserving strict graph, header, provenance and archive-symbol negatives. Scope FreeBSD loader assertions to their platform context. Retain original failed artifact identity and full command results before rethrowing early package-lane inspection failures. Keep launch/timeout results and optional resolved tool identity; preserve original errors if metadata or evidence persistence fails. Existing package/archive recovery workflows remain unchanged. Verified 47 audit fixtures, 12 affected lane tests, 7 FreeBSD recovery tests with real SCP peers, 1 selected-tool retention test and 2 catalog checks, plus changed Python compilation and diff whitespace. Native post-fix Windows/FreeBSD/Darwin checks remain incomplete and CI owned; no FreeBSD parser fix or CI-green claim.
1 parent 6918a23 commit e6d1da5

6 files changed

Lines changed: 299 additions & 23 deletions

‎scripts/audit-v3-installed-package.py‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import platform
1414
import re
1515
import shlex
16+
import shutil
1617
import subprocess
1718

1819

@@ -24,16 +25,32 @@ class AuditError(RuntimeError):
2425

2526

2627
def run(command, retain_output=True, **kwargs):
27-
result = subprocess.run(command, text=True, stdout=subprocess.PIPE,
28-
stderr=subprocess.PIPE, timeout=120, **kwargs)
29-
record = {'argv': command, 'returncode': result.returncode, 'stderr': result.stderr}
28+
record = {'argv': command, 'returncode': None}
29+
try:
30+
environment = kwargs.get('env')
31+
environment = os.environ if environment is None else environment
32+
selected = shutil.which(command[0], path=environment.get('PATH', os.defpath))
33+
if selected:
34+
record['resolved_tool'] = str(Path(selected).resolve())
35+
except (OSError, RuntimeError):
36+
pass # Optional tool metadata must not replace an inspection failure.
37+
if 'input' in kwargs:
38+
record['stdin'] = kwargs['input']
39+
COMMANDS.append(record)
40+
try:
41+
result = subprocess.run(command, text=True, stdout=subprocess.PIPE,
42+
stderr=subprocess.PIPE, timeout=120, **kwargs)
43+
except (OSError, subprocess.TimeoutExpired) as error:
44+
record['error'] = str(error)
45+
for key in ('stdout', 'stderr'):
46+
output = getattr(error, key, None)
47+
record[key] = output.decode(errors='replace') if isinstance(output, bytes) else output
48+
raise
49+
record.update(returncode=result.returncode, stderr=result.stderr)
3050
if retain_output:
3151
record['stdout'] = result.stdout
3252
else:
3353
record.update(stdout_sha256=hashlib.sha256(result.stdout.encode()).hexdigest(), stdout_bytes=len(result.stdout.encode()))
34-
if 'input' in kwargs:
35-
record['stdin'] = kwargs['input']
36-
COMMANDS.append(record)
3754
if result.returncode:
3855
raise AuditError(f'command failed: {shlex.join(command)}: {result.stderr[-3000:]}')
3956
return result.stdout
@@ -213,7 +230,11 @@ def visit(path):
213230
cached = platform_leaf and not Path(identity).exists()
214231
nodes.append({'id': identity, 'provenance': 'OS/API-set platform leaf' if cached else 'platform inspector',
215232
'sha256': None if cached else hashlib.sha256(Path(identity).read_bytes()).hexdigest()})
216-
dependencies = [] if cached else inspector(identity)
233+
try:
234+
dependencies = [] if cached else inspector(identity)
235+
except (AuditError, OSError, subprocess.TimeoutExpired) as error:
236+
error.failed_artifact = {'id': identity, 'sha256': nodes[-1]['sha256']}
237+
raise
217238
if not isinstance(dependencies, list):
218239
raise AuditError('incomplete inspector result: ' + path)
219240
for name, location in dependencies:

‎scripts/check-v3-installed-consumer.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import platform
1010
import shlex
1111
import shutil
12+
import subprocess
1213
import sys
1314

1415
sys.path.insert(0, str(Path(__file__).parent / 'lib'))
@@ -24,6 +25,21 @@
2425

2526

2627
def package_lane(args, source, output, mode, inventory):
28+
audit.COMMANDS.clear()
29+
try:
30+
return _package_lane(args, source, output, mode, inventory)
31+
except (audit.AuditError, OSError, subprocess.TimeoutExpired) as error:
32+
receipt = dict(violations=[str(error)], commands=audit.COMMANDS.copy())
33+
if hasattr(error, 'failed_artifact'):
34+
receipt['failed_artifact'] = error.failed_artifact
35+
try:
36+
(output / ('tls-' + mode) / 'audit-failure.json').write_text(json.dumps(receipt, indent=2) + '\n')
37+
except OSError as persistence_error:
38+
error.add_note('could not persist audit failure evidence: ' + str(persistence_error))
39+
raise
40+
41+
42+
def _package_lane(args, source, output, mode, inventory):
2743
lane = output / ('tls-' + mode)
2844
build, prefix, consumer_dir = lane / 'build', lane / 'prefix', lane / 'consumer'
2945
for directory in (build, prefix, consumer_dir):

‎scripts/test-v3-ci-workflows.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"""Workflow dependencies and trust boundaries; execution contracts live beside runners."""
33
from pathlib import Path
44
import glob
5+
import importlib.util
56
import json
67
import os
78
import re
@@ -276,6 +277,43 @@ def test_successful_evidence_recovery_merges_named_directory_in_place(self):
276277
self.assertEqual((evidence / 'platform/FreeBSD.tar.gz').read_bytes(), b'platform proof')
277278
self.assertFalse((evidence / folder).exists())
278279

280+
def test_early_inspection_receipt_survives_package_and_archive_failure_recovery(self):
281+
spec = importlib.util.spec_from_file_location('installed_failure_fixture', ROOT / 'scripts/test-v3-installed-consumer.py')
282+
fixture = importlib.util.module_from_spec(spec)
283+
spec.loader.exec_module(fixture)
284+
for name in self.workflows:
285+
with self.subTest(workflow=name), tempfile.TemporaryDirectory() as directory:
286+
root = Path(directory).resolve()
287+
runner = root / 'runner'
288+
package = name == self.workflows[0]
289+
folder = 'package-evidence' if package else 'platform-evidence'
290+
case = fixture.InstalledInspectionFailureTest()
291+
try:
292+
case.setUp()
293+
case.output = runner / ('package-evidence/matrix' if package else 'platform/matrix')
294+
case.output.mkdir(parents=True)
295+
error = case.fail_lane()
296+
self.assertEqual(str(error), 'empty ELF dependency inspection')
297+
receipt = case.output / 'tls-no/audit-failure.json'
298+
self.assertTrue(receipt.is_file(), 'failed package lane must leave recoverable inspection evidence')
299+
original = receipt.read_bytes()
300+
document = json.loads(original)
301+
self.assertEqual(document['commands'][-1]['stdout'], case.dynamic)
302+
self.assertEqual(document['violations'], [str(error)])
303+
# Execute the existing EXIT copyback before the existing
304+
# recovery helper's real SCP peers, replacing SSH only.
305+
run, steps = self.guest_run(name, runner)
306+
preamble = run.split('# Guest root', 1)[0]
307+
result = subprocess.run(['sh', '-eu', '-c', preamble + '\nexit 1'], cwd=root,
308+
capture_output=True, text=True, timeout=10)
309+
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
310+
relative = Path('matrix/tls-no/audit-failure.json' if package else 'platform/matrix/tls-no/audit-failure.json')
311+
self.assertEqual((root / folder / relative).read_bytes(), original)
312+
recovered = self.recover(root, steps, runner, dict(os.environ))
313+
self.assertEqual((recovered / relative).read_bytes(), original)
314+
finally:
315+
case.doCleanups()
316+
279317

280318
class WindowsGitPreflight(unittest.TestCase):
281319
def test_native_git_checkout_diagnostic_runs_before_provider_build(self):

‎scripts/test-v3-installed-consumer.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,108 @@ def setUp(self):
199199
self.addCleanup(observation.stop)
200200

201201

202+
class InstalledInspectionFailureTest(unittest.TestCase):
203+
"""Real lane/compile/inspector subprocesses with controlled producer tools.
204+
205+
Inspector text is deliberately unrecognized, not a FreeBSD format claim.
206+
"""
207+
def setUp(self):
208+
self.temp = tempfile.TemporaryDirectory()
209+
self.addCleanup(self.temp.cleanup)
210+
self.root = Path(self.temp.name).resolve()
211+
self.output = self.root / 'matrix'
212+
self.output.mkdir()
213+
self.source = self.root / 'source'
214+
(self.source / 'test/installed').mkdir(parents=True)
215+
(self.source / 'test/installed/package_consumer.cpp').write_text('int main() { return 0; }\n')
216+
(self.source / 'examples').mkdir()
217+
(self.source / 'examples/host_poll.hpp').write_text('// fixture\n')
218+
spec = importlib.util.spec_from_file_location('installed_inspection_runner',
219+
Path(__file__).with_name('check-v3-installed-consumer.py'))
220+
self.runner = importlib.util.module_from_spec(spec)
221+
spec.loader.exec_module(self.runner)
222+
self.args = SimpleNamespace(compiler=os.environ.get('CXX', 'c++'), cc='cc', make='fixture-make',
223+
jobs=1, prerequisite_prefix=[], pkg_config_dir=[], provider_prefix=self.root / 'provider',
224+
toolchain_prefix=None, optimized=False)
225+
self.dynamic = 'Dynamic section with unrecognized dependency syntax\n' + 'raw dynamic entry\n' * 400
226+
self.diagnostic = 'inspector diagnostic\n'
227+
self.inspector_exit = 0
228+
self.popen = subprocess.Popen
229+
environment = patch.dict(os.environ)
230+
environment.start()
231+
self.addCleanup(environment.stop)
232+
233+
def create_process(self, argv, **options):
234+
if argv[0] == 'bash':
235+
prefix = self.output / 'tls-no/prefix/lib/pkgconfig'
236+
code = f'from pathlib import Path; p=Path({str(prefix)!r}); p.mkdir(parents=True); (p/"libhttpserver.pc").write_text("fixture metadata")'
237+
elif argv[0] == 'fixture-make':
238+
code = ''
239+
elif argv[0] == 'pkg-config':
240+
value = str(self.output / 'tls-no/prefix/lib/pkgconfig') if '--variable=pcfiledir' in argv else ''
241+
code = f'print({value!r})'
242+
elif argv[0] == 'readelf':
243+
code = f'import sys; sys.stdout.write({self.dynamic!r}); sys.stderr.write({self.diagnostic!r}); sys.exit({self.inspector_exit})'
244+
else:
245+
return self.popen(argv, **options)
246+
return self.popen([sys.executable, '-c', code], **options)
247+
248+
def fail_lane(self):
249+
with patch.object(self.runner.audit.platform, 'system', return_value='FreeBSD'), \
250+
patch.object(subprocess, 'Popen', side_effect=self.create_process):
251+
with self.assertRaises(self.runner.audit.AuditError) as caught:
252+
self.runner.package_lane(self.args, self.source, self.output, 'no', matrix.load())
253+
return caught.exception
254+
255+
def test_early_graph_failure_persists_full_output_and_failed_consumer(self):
256+
self.runner.audit.COMMANDS.append({'argv': ['stale-previous-lane-command']})
257+
error = self.fail_lane()
258+
self.assertEqual(str(error), 'empty ELF dependency inspection')
259+
receipt_path = self.output / 'tls-no/audit-failure.json'
260+
self.assertTrue(receipt_path.is_file(), 'early graph failure must persist inspection evidence')
261+
receipt = json.loads(receipt_path.read_text())
262+
consumer = self.output / ('tls-no/consumer/package_consumer' + ('.exe' if os.name == 'nt' else ''))
263+
self.assertEqual(receipt['violations'], [str(error)])
264+
self.assertEqual(receipt['failed_artifact'], dict(id=str(consumer),
265+
sha256=hashlib.sha256(consumer.read_bytes()).hexdigest()))
266+
command = receipt['commands'][-1]
267+
self.assertEqual(len(receipt['commands']), 1)
268+
self.assertEqual(command['argv'], ['readelf', '-d', str(consumer)])
269+
self.assertEqual(command['returncode'], 0)
270+
self.assertEqual(command['stdout'], self.dynamic)
271+
self.assertEqual(command['stderr'], self.diagnostic)
272+
selected_tool = shutil.which('readelf')
273+
if selected_tool:
274+
self.assertEqual(command['resolved_tool'], str(Path(selected_tool).resolve()))
275+
else:
276+
self.assertNotIn('resolved_tool', command)
277+
self.assertFalse((self.output / 'tls-no/audit.json').exists())
278+
279+
def test_nonzero_inspector_remains_failed_with_actual_exit_evidence(self):
280+
self.inspector_exit = 7
281+
error = self.fail_lane()
282+
self.assertIn('command failed: readelf', str(error))
283+
receipt_path = self.output / 'tls-no/audit-failure.json'
284+
self.assertTrue(receipt_path.is_file(), 'nonzero inspector must persist its actual result')
285+
receipt = json.loads(receipt_path.read_text())
286+
self.assertEqual(receipt['violations'], [str(error)])
287+
self.assertEqual(receipt['commands'][-1]['returncode'], 7)
288+
self.assertEqual(receipt['commands'][-1]['stdout'], self.dynamic)
289+
self.assertEqual(receipt['commands'][-1]['stderr'], self.diagnostic)
290+
291+
def test_failure_receipt_write_error_keeps_original_graph_failure(self):
292+
real_write = Path.write_text
293+
def write(path, *args, **kwargs):
294+
if path.name == 'audit-failure.json':
295+
raise OSError('evidence disk unavailable')
296+
return real_write(path, *args, **kwargs)
297+
with patch.object(Path, 'write_text', write):
298+
error = self.fail_lane()
299+
self.assertEqual(str(error), 'empty ELF dependency inspection')
300+
self.assertEqual(error.__notes__, ['could not persist audit failure evidence: evidence disk unavailable'])
301+
self.assertEqual(self.runner.audit.COMMANDS[-1]['stdout'], self.dynamic)
302+
303+
202304
class MatrixTest(unittest.TestCase):
203305
def setUp(self):
204306
self.inventory = matrix.load()

0 commit comments

Comments
 (0)