From 127493045bb7a02136d05a89317c9be250a73ba2 Mon Sep 17 00:00:00 2001 From: ajhcs <176340565+ajhcs@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:36:18 +0000 Subject: [PATCH] Fix native parser portability on Windows --- .github/workflows/check.yml | 6 +- README.md | 2 +- docs/windows-runtime.md | 54 ++++++++ scripts/check.py | 2 +- tests/test_skill_runtime.py | 60 ++++++--- tests/test_sysml_portability.py | 231 ++++++++++++++++++++++++++++++++ verifier/sysml.py | 91 ++++++++++--- 7 files changed, 406 insertions(+), 40 deletions(-) create mode 100644 docs/windows-runtime.md create mode 100644 tests/test_sysml_portability.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 00db126..a03f6fb 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,10 +10,14 @@ permissions: jobs: test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-latest, windows-latest] python-version: ['3.10', '3.12'] + env: + PYTHONUTF8: "1" + PYTHONIOENCODING: utf-8 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: diff --git a/README.md b/README.md index 4e31dba..2f684ab 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ retain their current checks. Try the neutral fixture with The repository registers its three skills through symlinks under `.agents/skills/`, which Codex discovers when you open this project. They remain independently invocable and are scoped to this repository. Start a new project conversation if the skill selector has not refreshed. [Codex skill discovery](https://learn.chatgpt.com/docs/build-skills) documents repository scope and symlink support. -Each skill's `scripts/cse.py` launcher finds the shared parser/engine from its real location, so checks work from subdirectories as well as the repository root. The repository also includes a plugin manifest at `.codex-plugin/plugin.json`; installation through a public marketplace remains untested. Copying a skill folder alone preserves its version-1 helper, but does not install the shared SysML engine. Avoid duplicate installations. +Each skill's `scripts/cse.py` launcher finds the shared parser/engine from its real location, so checks work from subdirectories as well as the repository root. The repository also includes a plugin manifest at `.codex-plugin/plugin.json`; installation through a public marketplace remains untested. Copying a skill folder alone preserves its version-1 helper, but does not install the shared SysML engine. Avoid duplicate installations. [Windows runtime and installation boundaries](docs/windows-runtime.md) describe platform locking, UTF-8, checkout links, and the separate installed-plugin qualification. Example requests: diff --git a/docs/windows-runtime.md b/docs/windows-runtime.md new file mode 100644 index 0000000..3173bfa --- /dev/null +++ b/docs/windows-runtime.md @@ -0,0 +1,54 @@ +# Native runtime portability and installation boundary + +The shared native parser needs Python 3.10+, Java and javac21+, and the same pinned +pilot archive on both Linux and Windows. This change preserves parser serialization: +POSIX uses nonblocking `flock`; Windows uses a nonblocking one-byte `msvcrt` lock +at byte zero of the same persistent `parser.lock`. A contender must wait or time +out. Lock errors never fall back to running Java without a lock. The lock file +is opened without truncation and is not deleted or replaced while contenders +may be using it. File closure and explicit release apply on parser failure too. + +The existing timeout bounds lock acquisition and the Java invocation together. +Java's two-entry classpath uses the platform path separator, and source/result +pipes carry explicit UTF-8 bytes. Java launches with `-Dstdout.encoding=UTF-8` +and `-Dstderr.encoding=UTF-8`; Python strictly decodes both captured byte streams +on the calling thread. Invalid or missing streams produce a structured parser +error, including on nonzero exits. Paths with spaces remain individual subprocess +arguments; no shell command is constructed. + +Run the normal required suite after installing the documented pinned parser: + +```sh +python -X utf8 scripts/install_sysml_parser.py +python -X utf8 -B scripts/check.py --require-sysml +``` + +The new portability suite exercises real cross-process exclusion, timeout without +Java execution, release after exceptions/timeouts, preserved lock contents, +nonblocking Windows byte-zero adapter behavior, import without POSIX-only modules, +actual JVM stdout/stderr UTF-8 bytes, malformed transport errors, +and actual native acquisition of UTF-8 text in a path with spaces. The workflow +adds Windows alongside Ubuntu for both supported Python test versions. Hosted +workflow configuration alone is not a completed Windows test result; record the +actual run and revision before claiming qualification. + +On Windows, Git may retain tracked `.agents/skills` symlinks as small files naming +their intended targets. The registration test checks those records, but does not +claim such a file supplies project skill discovery. Runtime tests use the real +`skills//scripts/cse.py` directory when a directory registration is absent. +Installed plugins likewise require the complete real skill directories plus the +shared scripts, verifier, reference files and qualified presentation assets. +Do not overwrite existing legacy standalone skill files or customizations to +repair a checkout. Project discovery through actual links and installed-plugin +discovery remain distinct checks. + +Global installation is a separate qualification: use a clean accepted revision, +verify manifest/marketplace loading through supported clients, compare installed +payload hashes, run the installed launchers/native check, and confirm skill +discovery in a clean session. This document does not claim a marketplace install, +Windows availability, editor roundtrip or protected execution merely from local +source tests. Do not restart active sessions or replace unrelated registrations. + +References: Python [msvcrt locking](https://docs.python.org/3/library/msvcrt.html), +[flock](https://docs.python.org/3/library/fcntl.html), and +[platform path separators](https://docs.python.org/3/library/os.html#os.pathsep). diff --git a/scripts/check.py b/scripts/check.py index 8f221b8..f81c5b6 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -42,7 +42,7 @@ def main(): return 1 suites = ("test_develop_conops.py", "test_scenario_split.py", "test_concept.py", "test_verifier.py", "test_language.py", "test_language_oracle.py", "test_influence.py", "test_events.py", - "test_public_language_cases.py", "test_sysml.py", "test_sysml_views.py", + "test_public_language_cases.py", "test_sysml_portability.py", "test_sysml.py", "test_sysml_views.py", "test_sysml_public.py", "test_skill_runtime.py", "test_engineering_views.py", "test_visual_style.py", "test_visual_projection.py", "test_visual_labels.py") for suite in suites: result = subprocess.run([sys.executable, "-B", str(ROOT / "tests" / suite)], diff --git a/tests/test_skill_runtime.py b/tests/test_skill_runtime.py index beb4e59..58a81f4 100644 --- a/tests/test_skill_runtime.py +++ b/tests/test_skill_runtime.py @@ -1,5 +1,7 @@ """Real repo skill discovery paths and CLI use from another working directory.""" +import io +import importlib.util import json import os from pathlib import Path @@ -7,6 +9,8 @@ import sys import tempfile import unittest +from contextlib import redirect_stderr +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) @@ -16,31 +20,47 @@ PARSER_AVAILABLE = JAR.is_file() and (CLASSES / "SysmlBridge.class").is_file() +def launcher(name): + registered = ROOT / ".agents/skills" / name + # Installed plugins carry real skill directories. Preserve a Windows Git + # symlink-record checkout rather than rewriting its registration files. + base = registered if registered.is_dir() else ROOT / "skills" / name + return base / "scripts/cse.py" + + class RuntimeTests(unittest.TestCase): def test_installer_explains_unsupported_java_before_download_or_compile(self): - (ROOT / ".tmp").mkdir(exist_ok=True) - with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: - java = Path(directory) / "java" - java.write_text("#!/bin/sh\nprintf 'openjdk 17.0.9\\n'\n") - java.chmod(0o755) - result = subprocess.run([sys.executable, "-B", str(ROOT / "scripts/install_sysml_parser.py")], - env=dict(os.environ, PATH=directory), text=True, capture_output=True) - self.assertEqual(result.returncode, 1) - self.assertIn("requires Java 21+ and javac 21+", result.stderr) - self.assertNotIn("bad class file", result.stderr) + spec = importlib.util.spec_from_file_location("cse_test_installer", ROOT / "scripts/install_sysml_parser.py") + installer = importlib.util.module_from_spec(spec); spec.loader.exec_module(installer) + old_java = subprocess.CompletedProcess(["java", "--version"], 0, "openjdk 17.0.9", "") + errors = io.StringIO() + with patch.object(installer.subprocess, "run", return_value=old_java) as run, \ + patch.object(installer, "urlopen") as download, redirect_stderr(errors): + self.assertEqual(installer.main(), 1) + self.assertIn("requires Java 21+ and javac 21+", errors.getvalue()) + self.assertNotIn("bad class file", errors.getvalue()) + self.assertEqual(run.call_args.args[0], ["java", "--version"]) + self.assertEqual(run.call_count, 1) + download.assert_not_called() - def test_repo_skill_links_resolve_to_complete_skills(self): + def test_repo_skill_registrations_refer_to_complete_skills(self): for name in SKILLS: skill = ROOT / ".agents/skills" / name - self.assertTrue(skill.is_symlink()) - self.assertEqual(skill.resolve(), ROOT / "skills" / name) - self.assertTrue((skill / "SKILL.md").is_file()) + if skill.is_symlink(): + self.assertEqual(skill.resolve(), ROOT / "skills" / name) + self.assertTrue((skill / "SKILL.md").is_file()) + else: + # Windows Git may preserve a tracked link as its target text. + # This checks metadata, not project discovery through that file. + self.assertEqual(os.name, "nt") + self.assertEqual(skill.read_text().strip(), "../../skills/" + name) + self.assertTrue((ROOT / "skills" / name / "SKILL.md").is_file()) def test_each_independent_launcher_finds_the_same_engine_from_another_directory(self): (ROOT / ".tmp").mkdir(exist_ok=True) with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: for name in SKILLS: - result = subprocess.run([sys.executable, "-B", str(ROOT / ".agents/skills" / name / "scripts/cse.py"), + result = subprocess.run([sys.executable, "-B", str(launcher(name)), "runtime"], cwd=directory, text=True, capture_output=True) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(json.loads(result.stdout)["runtime_root"], str(ROOT)) @@ -55,8 +75,8 @@ def test_joint_concept_scenario_uses_one_model_and_exports_supported_tests(self) with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: local_model = Path(directory) / "user-model.sysml" local_model.write_bytes(model.read_bytes()) - launcher = ROOT / ".agents/skills/develop-operational-scenarios/scripts/cse.py" - result = subprocess.run([sys.executable, "-B", str(launcher), "testgen", "user-model.sysml", + entrypoint = launcher("develop-operational-scenarios") + result = subprocess.run([sys.executable, "-B", str(entrypoint), "testgen", "user-model.sysml", "--output", "synthetic-tests.json"], cwd=directory, text=True, capture_output=True) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -70,8 +90,8 @@ def test_joint_concept_scenario_uses_one_model_and_exports_supported_tests(self) @unittest.skipUnless(PARSER_AVAILABLE, "Install the pinned SysML parser") def test_plain_output_distinguishes_compilation_and_behavior_and_reports_parser_errors(self): - launcher = ROOT / ".agents/skills/develop-system-concept/scripts/cse.py" - result = subprocess.run([sys.executable, "-B", str(launcher), "check", + entrypoint = launcher("develop-system-concept") + result = subprocess.run([sys.executable, "-B", str(entrypoint), "check", str(ROOT / "examples/sysml/ack.sysml")], text=True, capture_output=True) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("no behavior evaluated", result.stdout) @@ -79,7 +99,7 @@ def test_plain_output_distinguishes_compilation_and_behavior_and_reports_parser_ with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: model = Path(directory) / "invalid.sysml" model.write_text("package Broken { part def ???; }") - invalid = subprocess.run([sys.executable, "-B", str(launcher), "check", str(model), + invalid = subprocess.run([sys.executable, "-B", str(entrypoint), "check", str(model), "--syntax-only"], text=True, capture_output=True) self.assertNotEqual(invalid.returncode, 0) self.assertIn("ERROR", invalid.stdout) diff --git a/tests/test_sysml_portability.py b/tests/test_sysml_portability.py new file mode 100644 index 0000000..c19fd74 --- /dev/null +++ b/tests/test_sysml_portability.py @@ -0,0 +1,231 @@ +"""Cross-process parser serialization and portable native invocation contracts.""" +from contextlib import contextmanager, ExitStack +from hashlib import sha256 +import errno +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch, Mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +import verifier.sysml as engine + + +def hold_lock(path, ready, release): + deadline = time.monotonic() + 8 + with engine._parser_lock(path, deadline, ['lock-worker'], 8): + ready.write_text('locked', encoding='utf-8') + while not release.exists(): + if time.monotonic() >= deadline: + raise RuntimeError('Test lock holder exceeded its deadline') + time.sleep(0.01) + + +class PortabilityTests(unittest.TestCase): + def setUp(self): + (ROOT / '.tmp').mkdir(exist_ok=True) + self.temp = tempfile.TemporaryDirectory(dir=ROOT / '.tmp', prefix='portable-') + self.addCleanup(self.temp.cleanup) + self.base = Path(self.temp.name) + self.sequence = 0 + + @contextmanager + def holder(self, path): + self.sequence += 1 + ready = self.base / ('ready' + str(self.sequence)) + release = self.base / ('release' + str(self.sequence)) + child = subprocess.Popen([sys.executable, '-B', str(Path(__file__)), '--hold-lock', + str(path), str(ready), str(release)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + encoding='utf-8') + try: + deadline = time.monotonic() + 5 + while not ready.exists(): + if child.poll() is not None: + self.fail('Lock worker exited: ' + child.communicate()[1]) + if time.monotonic() >= deadline: + self.fail('Lock worker did not acquire the lock') + time.sleep(0.01) + yield + finally: + release.write_text('release', encoding='utf-8') + try: + _, errors = child.communicate(timeout=5) + except subprocess.TimeoutExpired: + child.kill(); child.communicate() + self.fail('Lock worker did not stop') + self.assertEqual(child.returncode, 0, errors) + + @contextmanager + def parser_cache(self): + cache = self.base / 'parser cache with spaces' + classes = cache / 'classes'; classes.mkdir(parents=True) + (classes / 'SysmlBridge.class').write_bytes(b'compiled fixture') + source = cache / 'bridge.java'; source.write_bytes(b'fixture source') + stamp = classes / 'SysmlBridge.source.sha256' + stamp.write_text(sha256(source.read_bytes()).hexdigest(), encoding='utf-8') + jar = cache / 'parser jar.jar'; jar.write_bytes(b'fixture jar') + library = cache / 'library'; library.mkdir() + values = dict(CACHE=cache, CLASSES=classes, JAR=jar, LIBRARY=library, + BRIDGE_SOURCE=source, BRIDGE_STAMP=stamp) + with ExitStack() as stack: + for key, value in values.items(): + stack.enter_context(patch.object(engine, key, value)) + yield values + + def test_actual_process_exclusion_timeout_and_later_acquisition(self): + path = self.base / 'parser.lock' + with self.holder(path): + start = time.monotonic() + with self.assertRaises(subprocess.TimeoutExpired): + with engine._parser_lock(path, start + 0.15, ['java'], 0.15): + self.fail('Concurrent parser entered the critical section') + self.assertGreaterEqual(time.monotonic() - start, 0.12) + with engine._parser_lock(path, time.monotonic() + 2, ['java'], 2): + pass + + def test_lock_content_survives_and_exception_releases_other_process(self): + path = self.base / 'parser.lock'; path.write_bytes(b'preserve existing contents') + with self.assertRaisesRegex(RuntimeError, 'inside parser'): + with engine._parser_lock(path, time.monotonic() + 2, ['java'], 2): + raise RuntimeError('inside parser') + with self.holder(path): + pass + self.assertEqual(path.read_bytes(), b'preserve existing contents') + + def test_expired_deadline_does_not_enter_critical_section(self): + with self.assertRaises(subprocess.TimeoutExpired): + with engine._parser_lock(self.base / 'parser.lock', time.monotonic() - 1, ['java'], 1): + self.fail('Expired acquisition entered the parser') + + def test_lock_contention_never_launches_java(self): + with self.parser_cache() as values, self.holder(values['CACHE'] / 'parser.lock'): + with patch.object(engine.subprocess, 'run') as run: + with self.assertRaises(subprocess.TimeoutExpired): + engine._parser_output('package Example {}', 0.1) + run.assert_not_called() + + def test_command_preserves_space_paths_utf8_and_remaining_budget(self): + with self.parser_cache() as values: + completed = subprocess.CompletedProcess(['java'], 0, + '{"text":"caf\u00e9 \u00b7 \u03b1"}'.encode('utf-8'), b'') + with patch.object(engine.subprocess, 'run', return_value=completed) as run: + result = engine._parser_output('package Example { /* caf\u00e9 \u00b7 \u03b1 */ }', 2) + command = run.call_args.args[0]; kwargs = run.call_args.kwargs + self.assertEqual(command[command.index('-cp') + 1], + str(values['CLASSES']) + os.pathsep + str(values['JAR'])) + self.assertIn('-Dstdout.encoding=UTF-8', command) + self.assertIn('-Dstderr.encoding=UTF-8', command) + self.assertNotIn('text', kwargs) + self.assertNotIn('encoding', kwargs) + self.assertEqual(kwargs['input'], 'package Example { /* caf\u00e9 \u00b7 \u03b1 */ }'.encode('utf-8')) + self.assertEqual(result['text'], 'caf\u00e9 \u00b7 \u03b1') + self.assertGreater(kwargs['timeout'], 0) + self.assertLessEqual(kwargs['timeout'], 2) + self.assertFalse(kwargs.get('shell', False)) + + def test_malformed_transport_is_a_structured_error_on_zero_and_nonzero_exit(self): + cases = [(None, b''), (b'{}', None), (b'\xe9', b''), + (b'{}', b'\xe9'), ('{}', b''), (b'{}', '')] + with self.parser_cache(): + path = self.base / 'input.sysml' + path.write_text('package Example {}', encoding='utf-8') + for code in (0, 1): + for stdout, stderr in cases: + with self.subTest(code=code, stdout=stdout, stderr=stderr): + completed = subprocess.CompletedProcess(['java'], code, stdout, stderr) + with patch.object(engine.subprocess, 'run', return_value=completed): + report = engine.check_sysml(path, syntax_only=True) + self.assertEqual(report['status'], 'error', report) + self.assertEqual(report['findings'][0]['code'], 'input_or_parser') + + def test_invalid_json_and_parser_exit_are_structured_errors(self): + cases = [(0, b'not JSON', b''), (0, b'[]', b''), + (1, b'{}', 'caf\u00e9 \u00b7 \u03b1'.encode('utf-8'))] + with self.parser_cache(): + path = self.base / 'input.sysml' + path.write_text('package Example {}', encoding='utf-8') + for code, stdout, stderr in cases: + with self.subTest(code=code, stdout=stdout): + completed = subprocess.CompletedProcess(['java'], code, stdout, stderr) + with patch.object(engine.subprocess, 'run', return_value=completed): + report = engine.check_sysml(path, syntax_only=True) + self.assertEqual(report['status'], 'error', report) + self.assertEqual(report['findings'][0]['code'], 'input_or_parser') + if code: + self.assertIn('caf\u00e9 \u00b7 \u03b1', report['findings'][0]['detail']) + + @unittest.skipUnless(engine.JAR.is_file() and (engine.CLASSES / 'SysmlBridge.class').is_file(), + 'Install the pinned SysML parser') + def test_actual_java_stdout_and_stderr_use_utf8(self): + # Obtain the production launch options, then exercise both real JVM pipes. + with self.parser_cache(): + with patch.object(engine.subprocess, 'run', return_value= + subprocess.CompletedProcess(['java'], 0, b'{}', b'')) as run: + engine._parser_output('package Example {}', 2) + options = run.call_args.args[0][1:run.call_args.args[0].index('-cp')] + source = self.base / 'Utf8Probe.java' + source.write_text('public class Utf8Probe { public static void main(String[] args) { ' + 'System.out.print("caf\u00e9 \u00b7 \u03b1"); System.err.print("caf\u00e9 \u00b7 \u03b1"); }}', + encoding='utf-8') + compile_result = subprocess.run(['javac', '--release', '21', '-encoding', 'UTF-8', str(source)], + capture_output=True, timeout=30, check=False) + self.assertEqual(compile_result.returncode, 0, compile_result.stderr) + result = subprocess.run(['java', *options, '-cp', str(self.base), 'Utf8Probe'], + capture_output=True, timeout=30, check=False) + self.assertEqual(result.returncode, 0, result.stderr) + expected = 'caf\u00e9 \u00b7 \u03b1'.encode('utf-8') + self.assertEqual(result.stdout, expected) + self.assertEqual(result.stderr, expected) + + def test_java_timeout_releases_lock_to_another_process(self): + with self.parser_cache() as values: + with patch.object(engine.subprocess, 'run', side_effect=subprocess.TimeoutExpired(['java'], 1)): + with self.assertRaises(subprocess.TimeoutExpired): + engine._parser_output('package Example {}', 1) + with self.holder(values['CACHE'] / 'parser.lock'): + pass + + def test_windows_byte_zero_nonblocking_busy_and_fatal_error_contract(self): + backend = SimpleNamespace(LK_NBLCK=2, LK_UNLCK=0, locking=Mock()) + lock = SimpleNamespace(seek=Mock(), fileno=lambda: 42) + with patch.dict(sys.modules, {'msvcrt': backend}), patch.object(engine.os, 'name', 'nt'): + self.assertTrue(engine._try_parser_lock(lock)) + lock.seek.assert_called_with(0) + backend.locking.assert_called_with(42, 2, 1) + engine._release_parser_lock(lock) + backend.locking.assert_called_with(42, 0, 1) + backend.locking.side_effect = OSError(errno.EACCES, 'busy') + self.assertFalse(engine._try_parser_lock(lock)) + backend.locking.side_effect = OSError(errno.EBADF, 'bad descriptor') + with self.assertRaises(OSError): engine._try_parser_lock(lock) + + def test_module_import_does_not_require_posix_fcntl(self): + result = subprocess.run([sys.executable, '-B', '-c', + "import sys;sys.modules['fcntl']=None;import verifier.sysml"], + cwd=ROOT, capture_output=True, text=True, encoding='utf-8', timeout=10) + self.assertEqual(result.returncode, 0, result.stderr) + + @unittest.skipUnless(engine.JAR.is_file() and (engine.CLASSES / 'SysmlBridge.class').is_file(), + 'Install the pinned SysML parser') + def test_actual_native_utf8_document_in_space_path(self): + path = self.base / 'native model with spaces.sysml' + path.write_text('package UnicodeCase { doc /* caf\u00e9 \u00b7 \u03b1 */ part def Equipment; part unit : Equipment; }', + encoding='utf-8') + report = engine.check_sysml(path, syntax_only=True) + self.assertEqual(report['status'], 'pass', report) + self.assertTrue(any('caf\u00e9 \u00b7 \u03b1' in d.get('body', '') for d in report['inventory']['documents'])) + + +if __name__ == '__main__': + if len(sys.argv) == 5 and sys.argv[1] == '--hold-lock': + hold_lock(*(Path(p) for p in sys.argv[2:])) + else: + unittest.main() diff --git a/verifier/sysml.py b/verifier/sysml.py index 0c09dae..6d0741d 100644 --- a/verifier/sysml.py +++ b/verifier/sysml.py @@ -4,8 +4,10 @@ compiler inputs for the existing deterministic language checker. """ +from contextlib import contextmanager from hashlib import sha256 -import fcntl +import errno +import os import json import math from pathlib import Path @@ -119,6 +121,55 @@ def _locate(line, line_ranges): return None, line +def _try_parser_lock(lock): + """Acquire the shared parser lock without waiting or an unlocked fallback.""" + try: + if os.name == "nt": + import msvcrt + lock.seek(0) + msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1) + elif os.name == "posix": + import fcntl + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + else: + raise RuntimeError("Native parser locking is unsupported on this platform") + except OSError as exc: + if exc.errno in (errno.EACCES, errno.EAGAIN): + return False + raise + return True + + +def _release_parser_lock(lock): + if os.name == "nt": + import msvcrt + lock.seek(0) + msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1) + elif os.name == "posix": + import fcntl + fcntl.flock(lock, fcntl.LOCK_UN) + else: + raise RuntimeError("Native parser locking is unsupported on this platform") + + +@contextmanager +def _parser_lock(path, deadline, command, timeout): + # Do not truncate: Windows byte-range locks must coexist with other openers. + # Every Windows opener locks byte zero, including on an empty lock file. + with path.open("a+b") as lock: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(command, timeout) + if _try_parser_lock(lock): + break + time.sleep(min(0.05, max(0, deadline - time.monotonic()))) + try: + yield + finally: + _release_parser_lock(lock) + + def _parser_output(source, timeout): if not JAR.is_file() or not LIBRARY.is_dir() or not (CLASSES / "SysmlBridge.class").is_file(): raise RuntimeError("SysML pilot not installed; run python3 scripts/install_sysml_parser.py") @@ -127,27 +178,33 @@ def _parser_output(source, timeout): raise RuntimeError("SysML bridge source changed; rerun python3 scripts/install_sysml_parser.py") if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout <= 0 or timeout > 600: raise ValueError("Parser timeout must be between 0 and 600 seconds") - command = ["java", "-Xmx1200m", "-cp", f"{CLASSES}:{JAR}", "SysmlBridge", + command = ["java", "-Xmx1200m", "-Dstdout.encoding=UTF-8", "-Dstderr.encoding=UTF-8", + "-cp", os.pathsep.join((str(CLASSES), str(JAR))), "SysmlBridge", str(LIBRARY), "-"] CACHE.mkdir(parents=True, exist_ok=True) # The pilot may refresh a shared library index; serialize local parser runs. - with (CACHE / "parser.lock").open("w") as lock: - deadline = time.monotonic() + timeout - while True: - try: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except BlockingIOError: - if time.monotonic() >= deadline: - raise subprocess.TimeoutExpired(command, timeout) - time.sleep(min(0.05, max(0, deadline - time.monotonic()))) - remaining = max(0.001, deadline - time.monotonic()) - completed = subprocess.run(command, input=source, capture_output=True, text=True, timeout=remaining, - check=False) + deadline = time.monotonic() + timeout + with _parser_lock(CACHE / "parser.lock", deadline, command, timeout): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(command, timeout) + # Capture bytes: Windows reader-thread decoding failures can otherwise + # leave a pipe as None and bypass the structured parser error report. + completed = subprocess.run(command, input=source.encode("utf-8"), capture_output=True, + timeout=remaining, check=False) + streams = {} + for name in ("stdout", "stderr"): + raw = getattr(completed, name) + if not isinstance(raw, bytes): + raise RuntimeError(f"SysML parser returned an invalid {name} byte stream") + try: + streams[name] = raw.decode("utf-8") + except UnicodeError as exc: + raise RuntimeError(f"SysML parser returned non-UTF-8 {name}") from exc if completed.returncode: - raise RuntimeError(f"SysML parser exited {completed.returncode}: {completed.stderr[-500:]}") + raise RuntimeError(f"SysML parser exited {completed.returncode}: {streams['stderr'][-500:]}") try: - data = json.loads(completed.stdout) + data = json.loads(streams["stdout"]) except ValueError as exc: raise RuntimeError("SysML parser did not return a JSON result") from exc if not isinstance(data, dict):