Merge FEATURE-ci-bash-empty-arrays: preserve optional flags on legacy Bash

Author: etr

scripts/ci-freebsd-toolchain.py

@@ -63,8 +63,9 @@ def write_compiler(prefix, cxx):
     -static) rpath=() ;;
   esac
 done
+# Bash 3.2 nounset needs guarded empty arrays; absent flags must add zero arguments.
 exec ''' + shlex.join([cxx, '-nostdinc++', '-isystem', str(prefix / 'include/c++/v1')])
-                       + ' "$@" "${link_flags[@]}" "${rpath[@]}"\n')
+                       + ' "$@" ${link_flags[@]+"${link_flags[@]}"} ${rpath[@]+"${rpath[@]}"}\n')
     wrapper.chmod(0o755)
     return wrapper
 

scripts/run-v3-protocol-fuzz.sh

@@ -196,9 +196,10 @@ PYTHON
     fi
     link_flags=()
     if [ "$target" = quic_tls_callbacks ]; then link_flags+=("${provider_libs[@]}"); fi
-    printf '%q ' "$compiler" "${flags[@]}" "${target_flags[@]}" "${sources[@]}" "${link_flags[@]}" -o "$build/$target/fuzz" > "$build/$target/compile.sh"
+    # Bash 3.2 nounset needs guarded empty arrays; absent flags must add zero arguments.
+    printf '%q ' "$compiler" "${flags[@]}" ${target_flags[@]+"${target_flags[@]}"} "${sources[@]}" ${link_flags[@]+"${link_flags[@]}"} -o "$build/$target/fuzz" > "$build/$target/compile.sh"
     printf '\n' >> "$build/$target/compile.sh"
-    "$compiler" "${flags[@]}" "${target_flags[@]}" "${sources[@]}" "${link_flags[@]}" -o "$build/$target/fuzz" > "$build/$target/build.log" 2>&1
+    "$compiler" "${flags[@]}" ${target_flags[@]+"${target_flags[@]}"} "${sources[@]}" ${link_flags[@]+"${link_flags[@]}"} -o "$build/$target/fuzz" > "$build/$target/build.log" 2>&1
     printf '%q ' "$build/$target/fuzz" "$target_corpus" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed="$seed" -artifact_prefix="$build/$target/artifacts/" > "$build/$target/reproduce.sh"
     printf '\n' >> "$build/$target/reproduce.sh"
     "$build/$target/fuzz" "$target_corpus" -runs="$runs" -max_total_time="$seconds" -max_len="$max_len" -timeout=5 -rss_limit_mb=512 -seed="$seed" -artifact_prefix="$build/$target/artifacts/" > "$build/$target/run.log" 2>&1 || { echo "fuzz failure: $target; see $build/$target/run.log and artifacts" >&2; exit 1; }

scripts/test-v3-platform-ci.py

@@ -77,27 +77,25 @@ def test_compiler_wrapper_keeps_headers_and_runtime_through_clean_environment(se
         selector = self.selector()
         with tempfile.TemporaryDirectory() as directory:
             root = Path(directory)
-            compiler = root / 'compiler'
+            compiler = root / 'compiler with spaces'
             compiler.write_text('#!' + sys.executable + '\nimport json, sys\nprint(json.dumps(sys.argv[1:]))\n')
             compiler.chmod(0o755)
             prefix = root / 'selected sdk'
             wrapper = selector.write_compiler(prefix, str(compiler))
-            for mode in ('', '-c', '-E', '-S', '-fsyntax-only', '-r', '-Wl,-r', '-static'):
+            # The wrapper's env shebang must use the selected Bash even in the clean environment.
+            environment = {'PATH': str(Path(shutil.which('bash')).parent) + os.pathsep + os.defpath}
+            for mode in ('', '-c', '-E', '-S', '-fsyntax-only', '-r', '-Wl,-r', '--version', '-static', '-shared'):
                 with self.subTest(mode=mode):
                     args = ([mode] if mode else []) + ['path with spaces.cpp', '-o', 'path with spaces.o']
-                    result = subprocess.run([str(wrapper)] + args, env={'PATH': os.defpath}, capture_output=True, text=True)
+                    result = subprocess.run([str(wrapper)] + args, env=environment, capture_output=True, text=True)
                     self.assertEqual(result.returncode, 0, result.stderr)
                     actual = json.loads(result.stdout)
-                    self.assertIn('-nostdinc++', actual)
-                    self.assertEqual(actual[actual.index('-isystem') + 1], str(prefix / 'include/c++/v1'))
-                    self.assertIn('path with spaces.cpp', actual)
-                    if mode in ('-c', '-E', '-S', '-fsyntax-only', '-r', '-Wl,-r'):
-                        self.assertNotIn('-lc++', actual)
-                        self.assertFalse(any(flag.startswith('-Wl,-rpath,') for flag in actual))
-                    else:
-                        self.assertIn('-L' + str(prefix / 'lib'), actual)
-                        self.assertIn('-lc++', actual)
-                        self.assertEqual(any(flag.startswith('-Wl,-rpath,') for flag in actual), mode != '-static')
+                    expected = ['-nostdinc++', '-isystem', str(prefix / 'include/c++/v1')] + args
+                    if mode not in ('-c', '-E', '-S', '-fsyntax-only', '-r', '-Wl,-r', '--version'):
+                        expected += ['-nostdlib++', '-L' + str(prefix / 'lib'), '-lc++']
+                        if mode != '-static':
+                            expected += ['-Wl,-rpath,' + str(prefix / 'lib')]
+                    self.assertEqual(actual, expected)
 
 
 class SelectedPlatformEnvironment(unittest.TestCase):

specs/tasks/FEATURE-ci-bash-empty-arrays.md

@@ -0,0 +1,90 @@
+# FEATURE-ci-bash-empty-arrays: preserve optional compiler arguments on legacy Bash
+
+Status: Complete — local implementation and validation complete; native confirmation remains CI/v3-PR owned.
+Base: `v3` at `103b11222e547f60471f6d43daad2759b08a8922`.
+Plan: `.groundwork-plans/FEATURE-ci-bash-empty-arrays-plan.md` in the primary checkout.
+Journal: `.debug/ci-bash-empty-arrays-q9c5zit7/journal.md` in the primary checkout.
+
+Native Darwin archive run `40030063567`, job `120676108141`, demonstrates
+`link_flags[@]: unbound variable` in the protocol fuzz runner and the generated
+FreeBSD compiler wrapper, plus `rpath[@]: unbound variable` for static wrapper
+invocation. Both owners enable nounset and expand optional empty arrays.
+
+Action items:
+
+- [x] Read the assigned plan, literal journal, primary CLAUDE.md, applicable
+  AGENTS and v3 architecture; isolate from the exact local v3 base.
+- [x] Establish actual Bash 3.2-family RED with the production fuzz runner and
+  generated wrapper; pin the official source and retain argument evidence.
+- [x] Handle absent optional arrays portably while retaining zero optional
+  arguments, quoted paths, ordering, mode selection and failure propagation.
+- [x] Verify empty target flags, empty link flags and populated provider flags;
+  preserve fuzz instrumentation, bounds, corpus security and metadata.
+- [x] Run focused tests with legacy/current Bash and real SDK compile/link/run
+  cancellation checks using the existing private libc++20 SDK.
+- [x] Check shell/Python syntax and whitespace; review the complete owned diff.
+- [x] Caller: perform full Groundwork validation, finalization and merge.
+- [ ] Native Darwin/FreeBSD post-repair execution: CI/v3-PR owned, not local.
+
+Other closed or queued Darwin fixture and C++ failures remain outside this
+repair. No broad baseline, CI control, merge or publication belongs to this task.
+
+Implementation and evidence (2026-10-23):
+
+The fuzz runner guards optional `target_flags` and `link_flags` in both the
+recorded compile command and actual compiler invocation. The FreeBSD wrapper
+guards optional `link_flags` and `rpath`. Bash's conditional array expansion
+adds zero arguments for absent flags and retains each populated element as one
+argument. Nounset, all existing mode decisions, provider selection, source
+selection, instrumentation, limits, corpus validation and exit handling remain
+in place. No dependency version or host Bash changes are introduced.
+
+The task-local interpreter was built from the official GNU Bash 3.2.57 source,
+SHA256 `3fa9daf85ebf35068f090ce51283ddeeb3c75eb5bc70b1a4a7cb05868bfe06a4`.
+The GNU detached signature verifies with the maintainer's HTTPS-served key,
+fingerprint `7C0135FB088AAF6C66C650B9BB5869F064EA74AB` (no configured web-of-trust
+certification). The source needed yacc to regenerate its patched parser;
+Ubuntu bison was extracted under the task proof directory, without installation.
+Actual runtime: GNU Bash `3.2.57(2)-release` on Linux; comparison runtime:
+`5.2.21(1)-release`. Native Darwin's exact interpreter patch level was not
+captured, so this establishes the real 3.2-family behavior, not native success.
+
+Before production edits, legacy execution failed the H3/QPACK invocation at
+`link_flags[@]` and the HTTP/1 corpus invocation at `target_flags[@]`. The
+generated wrapper failed eight mode cases at empty `link_flags` or `rpath`,
+including `--version`. The current interpreter passed before the change.
+After the repair:
+
+- Protocol fuzz runner: 13 tests pass with each interpreter, covering empty
+  flags, populated provider flags with spaces, exact compile metadata, limits,
+  configuration failures, restored corpora and symlink rejection.
+- FreeBSD toolchain: 4 tests pass with each interpreter. Exact argument vectors
+  cover final/shared links, compile/preprocess/assembly/syntax/relocatable modes,
+  version and static modes, using compiler and SDK paths containing spaces.
+- Real local Clang18 builds and runs the production HTTP/1 head fuzzer with
+  unchanged sanitizer flags, runs/time bounds and a spaced build path under
+  each interpreter. The initial sandboxed capability run failed because
+  LeakSanitizer cannot run under ptrace; authorized execution outside that
+  sandbox preserves the sanitizer gate and reaches the expected legacy RED,
+  then GREEN. The failure receipt is retained.
+- Existing private libc++20 SDK: newly generated production wrappers compile,
+  link and run the production cancellation probe under each interpreter;
+  real compile/preprocess/assembly/syntax/version and relocatable checks pass.
+  Raw compiler arguments, dependencies and source hashes are retained. Linux's
+  forwarded `-Wl,-r` required caller flags `-no-pie -nostdlib` to suppress
+  executable defaults; the two initial task-probe setup failures are retained.
+  This is local Linux SDK selection proof, not native FreeBSD/libcxxrt ABI proof.
+- Shell syntax under both interpreters, Python compilation and
+  `git diff --check` pass. CLAUDE.md's C++ cpplint command does not apply to these
+  shell/Python changes; no Python/shell linter is configured there.
+
+Durable source pin, signature, interpreter, RED/GREEN logs, raw argument
+receipts, exact source snapshots and task-executor notes are retained in the
+primary checkout's `.debug/ci-bash-empty-arrays-q9c5zit7/implementation/`.
+The task-local build and SDK probes remain in `/tmp/ci-bash-empty-arrays-proof`.
+All other failure assessments remain closed with their prior uncertainty.
+Post-repair native execution belongs to CI/v3 PR; overall CI is not green.
+
+Final local validation/finalization (2026-10-23):
+
+Groundwork sealed exact clean implementation `118db3bf3ba3eed0dcfb2b202ba76a2323f99dbe` / tree `ea89ccad7217f2861864c91d98bb052741c39dcd`: nine fresh independent profiles approved, 1 iteration, 0 fixed, 0 unworked. Durable receipt: `.git/groundwork/validation/cdb4ee2aea69cc6a/e0be2ef7dab5001c/groundwork-validation-3cefc1a6aa3b8eb2c15f656700685e35`. Existing hash-matched interpreter, regression, real fuzz and SDK receipts were reused; no broader suite or unrelated diagnostic was repeated. The initial spec-artifact automatic approval rejection was resolved with an evidence-based approval explicitly limited to local acceptance, preserving historical failures and native limits; no approval blocker remains. Caller changes only this bookkeeping before serial merge/cleanup. Native Darwin/FreeBSD remains unexecuted after this repair and assigned to CI/v3 PR. Other established selected-tool/fixture-path setup repairs follow serially; the five C++ causes remain uncertain and closed to further probes pending a later decision after independent work. Overall CI is not green.

test/integ/protocol_fuzz_runner_test.py

@@ -96,9 +96,10 @@ def test_http3_qpack_build_and_invocation_contract(self):
         # This probe verifies shell arguments; the real sanitizer runs are separate.
         import json
         import os
+        import shlex
         with tempfile.TemporaryDirectory() as directory:
             location = pathlib.Path(directory)
-            compiler = location / 'compiler.py'
+            compiler = location / 'compiler with spaces.py'
             compiler.write_text('''#!/usr/bin/env python3
 import json, os, pathlib, sys
 if '--version' in sys.argv:
@@ -110,15 +111,15 @@ def test_http3_qpack_build_and_invocation_contract(self):
 output.chmod(0o755)
 ''')
             compiler.chmod(0o755)
-            config = location / 'native'
+            config = location / 'native config'
             config.mkdir()
             (config / 'config.h').write_text('/* TLS disabled */\n')
-            build = location / 'fuzz'
+            build = location / 'fuzz output'
             environment = dict(os.environ, FUZZ_ARGUMENT_LOG=str(location / 'compile.jsonl'),
                                FUZZ_RUN_LOG=str(location / 'run.jsonl'))
             result = subprocess.run([
                 'bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'),
-                '--build-dir', str(build), '--targets', 'http3_engine qpack',
+                '--build-dir', str(build), '--targets', 'http3_engine qpack http1_head',
                 '--compiler', str(compiler), '--native-config-dir', str(config),
                 '--runs', '9', '--seconds', '2'], env=environment,
                 capture_output=True, text=True, timeout=15)
@@ -128,8 +129,13 @@ def test_http3_qpack_build_and_invocation_contract(self):
             for target, macro, length in [('http3_engine', '-DHTTP3_ENGINE_LIBFUZZER', 4096),
                                           ('qpack', '-DQPACK_LIBFUZZER', 512)]:
                 arguments = next(args for args in builds if macro in args)
+                self.assertNotIn('', arguments)
                 self.assertIn('-fsanitize=fuzzer,address,undefined', arguments)
+                self.assertIn('-I' + str(config), arguments)
                 self.assertIn(str(ROOT / 'src/detail/http3_connection.cpp'), arguments)
+                self.assertEqual(arguments[-2:], ['-o', str(build / target / 'fuzz')])
+                self.assertEqual(shlex.split((build / target / 'compile.sh').read_text()),
+                                 [str(compiler)] + arguments)
                 self.assertTrue((build / target / 'sources.list').is_file())
                 invocation = next(args for args in runs if args[0] == str(build / target / 'fuzz'))
                 for flag in ('-runs=9', '-max_total_time=2', '-timeout=5', '-rss_limit_mb=512',
@@ -138,6 +144,57 @@ def test_http3_qpack_build_and_invocation_contract(self):
                 self.assertEqual(invocation[1], str(build / target / 'seeds'))
                 self.assertTrue(list((build / target / 'seeds').glob('*.seed')))
                 self.assertTrue((build / target / 'reproduce.sh').is_file())
+            head = next(args for args in builds if str(ROOT / 'test/fuzz/http1_head_fuzz.cpp') in args)
+            self.assertNotIn('', head)
+            self.assertIn('-fsanitize=fuzzer,address,undefined', head)
+            self.assertEqual(head[-2:], ['-o', str(build / 'http1_head/fuzz')])
+            self.assertEqual(shlex.split((build / 'http1_head/compile.sh').read_text()),
+                             [str(compiler)] + head)
+            self.assertEqual(len(builds), 4)  # Instrumentation probe and three selected targets.
+
+    def test_callback_provider_arguments_preserve_boundaries_and_link_order(self):
+        import json
+        import os
+        import shlex
+        with tempfile.TemporaryDirectory() as directory:
+            location = pathlib.Path(directory)
+            compiler = location / 'compiler with spaces'
+            compiler.write_text('''#!/usr/bin/env python3
+import json, os, pathlib, sys
+if '--version' in sys.argv:
+    print('test argument recorder'); sys.exit(0)
+with open(os.environ['FUZZ_ARGUMENT_LOG'], 'a') as log:
+    log.write(json.dumps(sys.argv[1:]) + '\\n')
+output = pathlib.Path(sys.argv[sys.argv.index('-o') + 1])
+output.write_text('#!/bin/sh\\nexit 0\\n')
+output.chmod(0o755)
+''')
+            compiler.chmod(0o755)
+            config = location / 'native config'
+            (config / 'src').mkdir(parents=True)
+            (config / 'config.h').write_text('#define NATIVE_V3_TLS 1\n')
+            cflags = ['-I' + str(location / 'provider headers'), '-DPROVIDER_VALUE=two words']
+            libs = ['-L' + str(location / 'provider libs'), '-lssl', '-lcrypto']
+            (config / 'src/Makefile').write_text('V3_TLS_CFLAGS = ' + shlex.join(cflags)
+                + '\nV3_TLS_LIBS = ' + shlex.join(libs) + '\n')
+            build = location / 'fuzz output'
+            log = location / 'compile.jsonl'
+            result = subprocess.run(['bash', str(ROOT / 'scripts/run-v3-protocol-fuzz.sh'),
+                '--build-dir', str(build), '--targets', 'quic_tls_callbacks',
+                '--compiler', str(compiler), '--native-config-dir', str(config),
+                '--runs', '1', '--seconds', '1'],
+                env=dict(os.environ, FUZZ_ARGUMENT_LOG=str(log)),
+                capture_output=True, text=True, timeout=15)
+            self.assertEqual(result.returncode, 0, result.stderr)
+            arguments = json.loads(log.read_text().splitlines()[-1])
+            self.assertNotIn('', arguments)
+            start = arguments.index('-DQUIC_TLS_CALLBACKS_LIBFUZZER')
+            self.assertEqual(arguments[start:start + 5],
+                ['-DQUIC_TLS_CALLBACKS_LIBFUZZER', '-I' + str(ROOT / 'test'), '-I' + str(config)] + cflags)
+            self.assertEqual(arguments[-5:], libs + ['-o', str(build / 'quic_tls_callbacks/fuzz')])
+            self.assertEqual(shlex.split((build / 'quic_tls_callbacks/compile.sh').read_text()),
+                             [str(compiler)] + arguments)
+            self.assertEqual((build / 'quic_tls_callbacks/provider.log').read_text().splitlines(), cflags + libs)
 
     def test_seed_and_corpus_arguments_reject_invalid_inputs_before_compilation(self):
         for seed in ('0', '-1', '4294967296', '$(touch marker)', '12x'):