Merge FEATURE-ci-darwin-canonical-fixtures: preserve native fixture and link-map identities

Author: etr

scripts/lib/release_package_provenance.py

@@ -5,6 +5,7 @@
 import re
 import shlex
 import os
+import platform
 from native_benchmark import require, verify_artifact, parse_json
 from package_commands import dependencies
 from package_matrix import check_dependencies, check_linkage
@@ -51,10 +52,18 @@ def bound(path, digest=None):
                 row['argv'][row['argv'].index('-o') + 1] == executable]
     require(len(selected) == 1, 'missing actual link command')
     link = selected[0]['argv']
-    require('-Wl,-Map,' + str(linkmap) in link, 'wrong link map command')
+    map_option = '-Wl,-map,' if platform.system() == 'Darwin' else '-Wl,-Map,'
+    require(map_option + str(linkmap) in link, 'wrong link map command')
     if linkage == 'static':
         archive = str(prefix / 'lib/libhttpserver.a')
-        contributors = {str(Path(name).resolve()) for name in re.findall(r'([^\s()]+libhttpserver\.a)\([^()\n]+\)', linkmap.read_text())}
+        text = linkmap.read_text()
+        if platform.system() == 'Darwin':
+            # ld64 lists contributing archive members in its object-file table.
+            table = text.partition('# Object files:\n')[2].split('\n# ', 1)[0]
+            names = re.findall(r'^\[\s*\d+\]\s+(.+libhttpserver\.a)(?:\[\d+\])?\([^()\n]+\)\s*$', table, re.M)
+        else:
+            names = re.findall(r'([^\s()]+libhttpserver\.a)\([^()\n]+\)', text)
+        contributors = {str(Path(name).resolve()) for name in names}
         require(archive in link and contributors == {archive}, 'different or noncontributing installed static archive')
     compile_rows = [row for row in commands if '-c' in row['argv'] and '-o' in row['argv'] and
                     row['argv'][row['argv'].index('-o') + 1] in link and

scripts/test-v3-native-benchmark.py

@@ -122,7 +122,7 @@ def command(argv, *_):
 
 class ReceiptTest(unittest.TestCase):
     def receipt(self, root, tls='no'):
-        root = Path(root)
+        root = Path(root).resolve()
         consumer = root / 'consumer'; consumer.mkdir()
         files = {'consumer.cpp': 'consumer', 'host_poll.hpp': 'poll', 'client.py': 'client',
                  'fixtures/root.pem': 'certificate', 'shared': 'shared binary', 'static': 'static binary',

scripts/test-v3-packaged-adapters.py

@@ -5,6 +5,9 @@
 import importlib.util
 import io
 import json
+import os
+import shlex
+import subprocess
 from pathlib import Path
 import sys
 import tarfile
@@ -26,7 +29,7 @@ def setUp(self):
         self.assertIsNotNone(adapter, 'packaged adapter implementation missing')
         self.temp = tempfile.TemporaryDirectory()
         self.addCleanup(self.temp.cleanup)
-        self.root = Path(self.temp.name)
+        self.root = Path(self.temp.name).resolve()
         self.source = self.root / 'source'
         self.source.mkdir()
         (self.source / 'unit.cpp').write_text('public source\n')
@@ -99,17 +102,22 @@ def provenance(self):
         unit = directory / 'consumer.cpp'; unit.write_text('public source\n')
         executable = directory / 'static'; executable.write_text('consumer')
         depfile = directory / 'static-0.d'; depfile.write_text(f'static.o: {unit} {header}\n')
-        linkmap = directory / 'static.map'; linkmap.write_text(str(library) + '(public.o)\nLOAD ' + str(library) + '\n')
+        linkmap = directory / 'static.map'
+        linkmap.write_text(('# Object files:\n[  1] ' + str(library) + '[1](public.o)\n# Sections:\n')
+                           if sys.platform == 'darwin' else str(library) + '(public.o)\nLOAD ' + str(library) + '\n')
+        compiler = os.environ.get('CXX', 'c++')
+        compiler_argv = shlex.split(compiler)
+        map_option = '-Wl,-map,' if sys.platform == 'darwin' else '-Wl,-Map,'
         obj = directory / 'static-0.o'; obj.write_text('object')
         log = self.root / 'command.log'; log.write_text('compile/link ok')
-        commands = [dict(argv=['c++', '-std=c++20', '-MMD', '-MF', str(depfile), '-c', str(unit), '-o', str(obj)],
+        commands = [dict(argv=compiler_argv + ['-std=c++20', '-MMD', '-MF', str(depfile), '-c', str(unit), '-o', str(obj)],
                          exit=0, log=str(log), sha256=bench.artifact(log)['sha256'], bytes=log.stat().st_size),
-                    dict(argv=['c++', str(obj), str(library), '-Wl,-Map,' + str(linkmap), '-o', str(executable)],
+                    dict(argv=compiler_argv + [str(obj), str(library), map_option + str(linkmap), '-o', str(executable)],
                          exit=0, log=str(log), sha256=bench.artifact(log)['sha256'], bytes=log.stat().st_size)]
         compiled = dict(private_test_headers=False, sources={str(unit): bench.artifact(unit)['sha256']},
                         dependencies=[str(unit), str(header)], dependency_sha256={str(p): bench.artifact(p)['sha256'] for p in (unit, header)},
                         artifact_sha256=bench.artifact(executable)['sha256'], link_map=str(linkmap))
-        identity = dict(source=str(self.source), build=str(self.root / 'build'), prefix=str(prefix), compiler='c++',
+        identity = dict(source=str(self.source), build=str(self.root / 'build'), prefix=str(prefix), compiler=compiler,
                         archive={'members': {'test/installed/consumer.cpp': bench.artifact(unit)['sha256']}},
                         package={str(p): bench.artifact(p)['sha256'] for p in (header, library)})
         artifacts = {str(p): bench.artifact(p) for p in directory.iterdir() if p.is_file()}
@@ -154,6 +162,29 @@ def test_empty_linkmap_and_foreign_dependency_are_rejected(self):
         artifacts[compiled['link_map']] = bench.artifact(compiled['link_map'])
         with self.assertRaises(ValueError): adapter.validate_compile(compiled, 'static', identity, artifacts, commands)
 
+    def test_darwin_link_map_requires_actual_selected_archive_contribution(self):
+        from unittest.mock import patch
+        compiled, identity, artifacts, commands = self.provenance()
+        linkmap = Path(compiled['link_map'])
+        archive = str(Path(identity['prefix']) / 'lib/libhttpserver.a')
+        foreign = str(self.root / 'foreign/libhttpserver.a')
+        commands[1]['argv'][-3] = '-Wl,-map,' + str(linkmap)
+        with patch('platform.system', return_value='Darwin'):
+            for member in (archive + '(public.o)', archive + '[12](public.o)'):
+                linkmap.write_text('# Path: consumer\n# Object files:\n[  0] linker synthesized\n[  1] ' + member + '\n# Sections:\n# Symbols:\n')
+                artifacts[str(linkmap)] = bench.artifact(linkmap)
+                adapter.validate_compile(compiled, 'static', identity, artifacts, commands)
+            for body in ('', 'LOAD ' + archive, archive + '(public.o)',
+                         '# Object files:\n[  1] ' + foreign + '[1](public.o)\n# Sections:\n',
+                         '# Object files:\n[  1] consumer.o\n# Sections:\n' + archive + '(unused.o)',
+                         '# Object files:\n[  1] ' + archive + '[1](public.o)\n[  2] ' + foreign + '[1](other.o)\n# Sections:\n'):
+                linkmap.write_text(body); artifacts[str(linkmap)] = bench.artifact(linkmap)
+                with self.subTest(body=body), self.assertRaisesRegex(ValueError, 'static archive'):
+                    adapter.validate_compile(compiled, 'static', identity, artifacts, commands)
+            commands[1]['argv'][-3] = '-Wl,-Map,' + str(linkmap)
+            with self.assertRaisesRegex(ValueError, 'link map command'):
+                adapter.validate_compile(compiled, 'static', identity, artifacts, commands)
+
     def test_artifact_catalog_rejects_pruning_and_command_tampering(self):
         from release_package_provenance import catalog, validate_catalog
         output = self.root / 'evidence'; output.mkdir()
@@ -363,6 +394,23 @@ def test_each_public_family_requires_its_compile_and_dependency_records(self):
                         bad = copy.deepcopy(child); bad.pop('independent_client')
                         with self.assertRaises(ValueError): provenance.validate_child_provenance(bad, identity, {}, [], family)
 
+    def test_static_archive_fixture_uses_selected_quoted_compiler(self):
+        compiler = shlex.split(os.environ.get('CXX', 'c++'))
+        wrapper = self.root / 'selected compiler.py'
+        marker = self.root / 'compiler-commands.jsonl'
+        wrapper.write_text('import json, os, sys\n'
+            'with open(' + repr(str(marker)) + ', "a") as stream: stream.write(json.dumps(sys.argv[1:]) + "\\n")\n'
+            'argv = ' + repr(compiler) + '\nos.execvp(argv[0], argv + sys.argv[1:])\n')
+        result = subprocess.run([sys.executable, str(Path(__file__).resolve()),
+            'AdapterContracts.test_unused_selected_static_archive_cannot_supply_link_identity'],
+            env=dict(os.environ, CXX=shlex.join([sys.executable, str(wrapper)])),
+            capture_output=True, text=True, timeout=30)
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        self.assertTrue(marker.is_file(), 'archive fixture bypassed the selected compiler')
+        commands = [json.loads(line) for line in marker.read_text().splitlines()]
+        self.assertEqual(sum('-c' in argv for argv in commands), 3)
+        self.assertEqual(sum('-c' not in argv for argv in commands), 2)
+
     def test_unused_selected_static_archive_cannot_supply_link_identity(self):
         import subprocess
         compiled, identity, artifacts, commands = self.provenance()
@@ -377,16 +425,17 @@ def test_unused_selected_static_archive_cannot_supply_link_identity(self):
             source = folder / 'implementation.cpp'; source.write_text('int implementation() { return %d; }\n' % value)
             obj = folder / 'implementation.o'; archive = folder / 'libhttpserver.a'
             archive.unlink(missing_ok=True)
-            subprocess.run(['c++', '-c', str(source), '-o', str(obj)], check=True, capture_output=True)
+            subprocess.run(shlex.split(identity['compiler']) + ['-c', str(source), '-o', str(obj)], check=True, capture_output=True)
             subprocess.run(['ar', 'rcs', str(archive), str(obj)], check=True, capture_output=True)
             libraries.append(str(archive))
-        commands[0]['argv'].insert(2, '-I' + str(Path(identity['prefix']) / 'include'))
-        commands[1]['argv'][2:3] = libraries
-        for command in commands: subprocess.run(command['argv'], check=True, capture_output=True)
+        from package_commands import Commands
+        from package_integration import compile_probe
+        command = Commands(directory)
+        compiled = compile_probe(identity['compiler'], [unit], ['-I' + str(Path(identity['prefix']) / 'include')],
+            libraries, directory / 'static', command, os.environ, self.source,
+            Path(identity['build']), Path(identity['prefix']), False)
+        commands = command.records
         self.assertEqual(subprocess.run([str(directory / 'static')]).returncode, 2)
-        compiled['sources'] = {str(unit): bench.artifact(unit)['sha256']}
-        compiled['dependency_sha256'] = {str(p): bench.artifact(p)['sha256'] for p in (unit, header)}
-        compiled['artifact_sha256'] = bench.artifact(directory / 'static')['sha256']
         identity['archive']['members']['test/installed/consumer.cpp'] = compiled['sources'][str(unit)]
         identity['package'] = {p: bench.artifact(p)['sha256'] for p in identity['package']}
         artifacts = {str(p): bench.artifact(p) for p in directory.iterdir() if p.is_file()}

specs/tasks/FEATURE-ci-darwin-canonical-fixtures.md

@@ -0,0 +1,38 @@
+# FEATURE-ci-darwin-canonical-fixtures
+
+Status: Complete (local validation and finalization; native CI pending)
+Base: local v3 `1fa030dd528e67218d7c0a6957371ba246679e96`
+
+Repair the exact known Darwin Python fixture root aliases, executable discovery,
+selected compiler and native linker-map setup failures. Keep original child names,
+symlink/tampering controls and production provenance policies intact.
+
+- [x] Reproduce known identity failures with real symlinked TMPDIR and actual validators.
+- [x] Resolve affected fixture roots before constructing identities/catalogs.
+- [x] Discover the success executable and use the selected compiler/native map format.
+- [x] Verify affected suites, syntax and diff; preserve negative controls.
+- [x] Caller validation and finalization.
+
+Evidence: primary `.debug/ci-darwin-canonical-fixtures-2shorq1l/journal.md`.
+Native post-fix Darwin, Windows and FreeBSD execution remains CI/v3 PR owned.
+No C++ investigations, runtime changes, full suite, SDK/Bash rebuilds or CI controls.
+
+Caller-approved owner expansion: `scripts/lib/release_package_provenance.py`
+`validate_compile` now recognizes the native Darwin linker option and contributing
+object-file table entries. GNU handling and exact installed archive/security bindings
+are preserved; actual real compilation uses unchanged `package_integration.compile_probe`.
+
+Verification: all six affected suites pass locally (155 tests including six real
+symlinked TMPDIR regressions), plus final focused reruns after test refactoring.
+Python byte-compilation and `git diff --check` pass. No standalone Python linter is
+documented; the repository cpplint command applies to C++, which is unchanged.
+RED/GREEN logs cover each known alias symptom, selected executable/compiler and native
+map grammar. No authentic retained Darwin map exists in the selected evidence tree;
+small ld64 format tests prove parser behavior only. Real local GNU map/archive controls
+prove foreign-first rejection and selected-only acceptance. Native post-fix platform
+checks remain incomplete and CI/v3 PR owned. Older C++ uncertainties remain closed.
+
+Groundwork validation `13002c772f224b35f4b6506f5647d66b` passed in one
+iteration, nine independent profiles approved, zero fixes and zero unworked
+findings; exact implementation HEAD `9ed911145c4bbb0eda9fdd1e5988c71373cb66c3`.
+Overall CI remains unsuccessful pending fresh native execution.

test/integ/release_artifact_test.py

@@ -26,7 +26,7 @@ def setUpClass(cls):
     def setUp(self):
         self.assertTrue(HELPER.is_file(), 'verified release-artifact identity helper missing')
         self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
-        self.directory = Path(self.tmp.name)
+        self.directory = Path(self.tmp.name).resolve()
         self.source = self.directory / 'source'; self.source.mkdir()
         (self.source / 'configure.ac').write_text('packaged source')
         self.archive = self.directory / 'release.tar.gz'; self.archive.write_bytes(b'archive identity')
@@ -111,4 +111,44 @@ def test_distribution_adapters_accept_verified_identity_instead_of_git(self):
             self.assertIn('release_artifact.py', (ROOT / 'scripts' / name).read_text())
 
 
+class CanonicalFixtureContracts(unittest.TestCase):
+    def check_fixture(self, script, *cases):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory).resolve()
+            target = root / 'native-tmp'; target.mkdir()
+            alias = root / 'tmp'; alias.symlink_to(target, target_is_directory=True)
+            self.assertNotEqual(alias, alias.resolve())
+            result = subprocess.run([sys.executable, str(ROOT / script), *cases],
+                                    env=dict(os.environ, TMPDIR=str(alias), TEMP=str(alias), TMP=str(alias)),
+                                    capture_output=True, text=True, timeout=60)
+            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+    def test_archive_parity_through_symlinked_tmpdir(self):
+        self.check_fixture('test/integ/release_artifact_test.py', 'ArchiveContracts')
+
+    def test_release_execution_through_symlinked_tmpdir(self):
+        self.check_fixture('test/integ/release_gate_runner_test.py',
+            'ReleaseContracts.test_packaged_families_execute_both_tls_lanes_and_independent_full_validation',
+            'ReleaseContracts.test_packaged_family_cannot_pass_when_independent_validation_rejects',
+            'ReleaseContracts.test_failed_stage_retains_its_original_command_log',
+            'ReleaseContracts.test_package_receipt_survives_later_native_build_and_check',
+            'ReleaseContracts.test_cxx_gate_bounds_each_native_test_before_waiting_for_aggregate')
+
+    def test_benchmark_provenance_through_symlinked_tmpdir(self):
+        self.check_fixture('scripts/test-v3-native-benchmark.py', 'ReceiptTest')
+
+    def test_public_compile_and_receipts_through_symlinked_tmpdir(self):
+        self.check_fixture('scripts/test-v3-packaged-adapters.py',
+            'AdapterContracts.test_real_public_compile_artifacts_are_rechecked',
+            'AdapterContracts.test_unused_selected_static_archive_cannot_supply_link_identity',
+            'AdapterContracts.test_h1_requires_exact_versions_and_two_persistent_pipeline_responses',
+            'AdapterContracts.test_unsafe_duplicate_and_symlink_archive_members_are_rejected')
+
+    def test_selected_windows_shell_through_symlinked_tmpdir(self):
+        self.check_fixture('test/integ/release_ci_test.py', 'RequestShellContracts')
+
+    def test_platform_origin_and_tool_aliases_through_symlinked_tmpdir(self):
+        self.check_fixture('test/integ/release_platform_bundle_test.py', 'PlatformBundles')
+
+
 if __name__ == '__main__': unittest.main()

test/integ/release_ci_test.py

@@ -21,7 +21,7 @@ def setUp(self):
         import subprocess
         self.temp = tempfile.TemporaryDirectory()
         self.addCleanup(self.temp.cleanup)
-        self.root = Path(self.temp.name)
+        self.root = Path(self.temp.name).resolve()
         self.selected = self.root / 'MSYS tools'
         self.selected.mkdir()
         self.bash = self.selected / ('bash.exe' if os.name == 'nt' else 'bash')