Skip to content

test(windows): launch the source CLI through the shipped host entry #6406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
huangruiteng merged 2 commits into from
Oct 14, 2026
Merged

test(windows): launch the source CLI through the shipped host entry #6406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cda18ef
test(windows): launch the source CLI through the shipped host entry
JasonBuildAI Oct 14, 2026
320762c
test(windows): harden the source CLI helper after review
JasonBuildAI Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 3 additions & 15 deletions tests/capabilities/test_benchmark_adaptive_concurrency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from __future__ import annotations

import json
import subprocess
from datetime import UTC, datetime, timedelta
from pathlib import Path

Expand All @@ -16,9 +15,7 @@
release_benchmark_case,
tune_benchmark_concurrency_target,
)

REPO_ROOT = Path(__file__).resolve().parents[2]

from tests.source_cli import run_source_cli

def _feedback(
now: datetime,
Expand Down Expand Up @@ -380,7 +377,6 @@ def test_cli_tune_preview_then_execute(tmp_path: Path) -> None:
)
headroom_path.write_text(json.dumps(_headroom(now)), encoding="utf-8")
base = [
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"concurrency-tune",
"--goal-id",
Expand All @@ -395,21 +391,13 @@ def test_cli_tune_preview_then_execute(tmp_path: Path) -> None:
"json",
]

preview = subprocess.run(
base, cwd=REPO_ROOT, text=True, capture_output=True, check=True
)
preview = run_source_cli(base, check=True)
assert json.loads(preview.stdout)["dry_run"] is True
assert (
read_benchmark_concurrency_envelope(path)["config"]["target_active_cases"] == 3
)

executed = subprocess.run(
[*base, "--execute"],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
executed = run_source_cli([*base, "--execute"], check=True)
payload = json.loads(executed.stdout)
assert payload["action"] == "increase"
assert payload["write_performed"] is True
Expand Down
17 changes: 3 additions & 14 deletions tests/capabilities/test_benchmark_behavior_finding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

import copy
import json
import subprocess
from pathlib import Path

import pytest

Expand All @@ -17,6 +15,7 @@
read_benchmark_upload_receipt,
simulate_benchmark_upload,
)
from tests.source_cli import run_source_cli


def finding():
Expand Down Expand Up @@ -232,10 +231,8 @@ def test_revision_requires_explicit_supersession_preserving_finding(tmp_path):
def test_cli_projects_finding_store(tmp_path):
store = tmp_path / "records.jsonl"
simulate_benchmark_upload(store, envelope(), execute=True)
repo = Path(__file__).resolve().parents[2]
result = subprocess.run(
result = run_source_cli(
[
str(repo / "scripts/loopx"),
"benchmark",
"behavior-report",
"--store",
Expand All @@ -247,9 +244,6 @@ def test_cli_projects_finding_store(tmp_path):
"--format",
"json",
],
cwd=repo,
capture_output=True,
text=True,
check=True,
)
assert (
Expand All @@ -259,10 +253,8 @@ def test_cli_projects_finding_store(tmp_path):


def test_documented_example_upload_envelope_cli():
repo = Path(__file__).resolve().parents[2]
result = subprocess.run(
result = run_source_cli(
[
str(repo / "scripts/loopx"),
"benchmark",
"upload-envelope",
"--payload-json",
Expand All @@ -286,9 +278,6 @@ def test_documented_example_upload_envelope_cli():
"--format",
"json",
],
cwd=repo,
text=True,
capture_output=True,
check=True,
)
assert json.loads(result.stdout) == envelope()
51 changes: 8 additions & 43 deletions tests/capabilities/test_benchmark_concurrency_envelope.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from __future__ import annotations

import json
import subprocess
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from pathlib import Path
Expand All @@ -20,8 +19,7 @@
release_benchmark_case,
)
from loopx.cli import main

REPO_ROOT = Path(__file__).resolve().parents[2]
from tests.source_cli import run_source_cli


def _connected_goal_registries(tmp_path: Path) -> tuple[Path, Path]:
Expand Down Expand Up @@ -434,7 +432,6 @@ def test_cli_previews_config_and_requires_admission_before_launch(
tmp_path: Path,
) -> None:
base = [
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"concurrency-configure",
"--goal-id",
Expand All @@ -457,30 +454,17 @@ def test_cli_previews_config_and_requires_admission_before_launch(
path = default_benchmark_concurrency_envelope_path(
project=tmp_path, goal_id="fixture-goal"
)
preview = subprocess.run(
base,
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
preview = run_source_cli(base, check=True)
assert json.loads(preview.stdout)["dry_run"] is True
assert not path.exists()

configured = subprocess.run(
[*base, "--execute"],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
configured = run_source_cli([*base, "--execute"], check=True)
configured_payload = json.loads(configured.stdout)
assert configured_payload["write_performed"] is True
assert str(tmp_path) not in configured.stdout

admitted = subprocess.run(
admitted = run_source_cli(
[
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"concurrency-admit",
"--goal-id",
Expand All @@ -497,16 +481,12 @@ def test_cli_previews_config_and_requires_admission_before_launch(
"--format",
"json",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
assert json.loads(admitted.stdout)["admitted"] is True

shown = subprocess.run(
shown = run_source_cli(
[
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"concurrency-status",
"--goal-id",
Expand All @@ -516,9 +496,6 @@ def test_cli_previews_config_and_requires_admission_before_launch(
"--format",
"json",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
payload = json.loads(shown.stdout)
Expand All @@ -531,9 +508,8 @@ def test_cli_previews_config_and_requires_admission_before_launch(
def test_cli_resource_headroom_gate_requires_fresh_typed_receipt(
tmp_path: Path,
) -> None:
configure = subprocess.run(
configure = run_source_cli(
[
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"concurrency-configure",
"--goal-id",
Expand All @@ -551,9 +527,6 @@ def test_cli_resource_headroom_gate_requires_fresh_typed_receipt(
"--format",
"json",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
assert (
Expand All @@ -563,9 +536,8 @@ def test_cli_resource_headroom_gate_requires_fresh_typed_receipt(
is True
)

missing = subprocess.run(
missing = run_source_cli(
[
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"concurrency-admit",
"--goal-id",
Expand All @@ -582,9 +554,6 @@ def test_cli_resource_headroom_gate_requires_fresh_typed_receipt(
"--format",
"json",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
)
assert missing.returncode == 2
Expand All @@ -603,9 +572,8 @@ def test_cli_resource_headroom_gate_requires_fresh_typed_receipt(
),
encoding="utf-8",
)
admitted = subprocess.run(
admitted = run_source_cli(
[
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"concurrency-admit",
"--goal-id",
Expand All @@ -624,9 +592,6 @@ def test_cli_resource_headroom_gate_requires_fresh_typed_receipt(
"--format",
"json",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
payload = json.loads(admitted.stdout)
Expand Down
51 changes: 7 additions & 44 deletions tests/capabilities/test_benchmark_experiment_board.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import copy
import itertools
import json
import subprocess
from pathlib import Path

import pytest
Expand All @@ -25,9 +24,7 @@
render_capability_detail_markdown,
)
from loopx.cli import main

REPO_ROOT = Path(__file__).resolve().parents[2]

from tests.source_cli import run_source_cli

def _row(
*,
Expand Down Expand Up @@ -866,7 +863,6 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None:
row_path = tmp_path / "row.json"
row_path.write_text(json.dumps(_baseline()), encoding="utf-8")
command = [
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"experiment-board-upsert",
"--goal-id",
Expand All @@ -879,13 +875,7 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None:
"json",
]

preview = subprocess.run(
command,
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
preview = run_source_cli(command, check=True)
preview_payload = json.loads(preview.stdout)
ledger = default_benchmark_experiment_board_path(
project=tmp_path, goal_id="fixture-goal"
Expand All @@ -894,22 +884,15 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None:
assert preview_payload["write"]["write_performed"] is False
assert not ledger.exists()

executed = subprocess.run(
[*command, "--execute"],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
executed = run_source_cli([*command, "--execute"], check=True)
executed_payload = json.loads(executed.stdout)
assert executed_payload["write"]["status"] == "inserted"
assert executed_payload["path_recorded"] is False
assert str(tmp_path) not in executed.stdout
assert ledger.exists()

shown = subprocess.run(
shown = run_source_cli(
[
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"experiment-board-show",
"--goal-id",
Expand All @@ -919,9 +902,6 @@ def test_cli_previews_then_writes_and_reads_board(tmp_path: Path) -> None:
"--format",
"json",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
shown_payload = json.loads(shown.stdout)
Expand All @@ -942,9 +922,8 @@ def test_cli_show_projects_factorial_contrast_from_explicit_contract(
contract_path = tmp_path / "four-arm-contract.json"
contract_path.write_text(json.dumps(_four_arm_contract()), encoding="utf-8")

shown = subprocess.run(
shown = run_source_cli(
[
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"experiment-board-show",
"--goal-id",
Expand All @@ -956,9 +935,6 @@ def test_cli_show_projects_factorial_contrast_from_explicit_contract(
"--format",
"json",
],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)

Expand Down Expand Up @@ -986,7 +962,6 @@ def test_cli_reconciles_provider_ledgers_without_recording_paths(
_baseline(observed_at="2026-08-18T00:20:00+00:00"),
)
command = [
str(REPO_ROOT / "scripts/loopx"),
"benchmark",
"experiment-board-reconcile",
"--goal-id",
Expand All @@ -1004,26 +979,14 @@ def test_cli_reconciles_provider_ledgers_without_recording_paths(
project=tmp_path, goal_id="fixture-goal"
)

preview = subprocess.run(
command,
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
preview = run_source_cli(command, check=True)
preview_payload = json.loads(preview.stdout)
assert preview_payload["summary"]["run_count"] == 1
assert preview_payload["write"]["status"] == "preview_reconciled"
assert preview_payload["write"]["inserted"] == 1
assert not ledger.exists()

executed = subprocess.run(
[*command, "--execute"],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=True,
)
executed = run_source_cli([*command, "--execute"], check=True)
executed_payload = json.loads(executed.stdout)
assert executed_payload["write"]["status"] == "reconciled"
assert executed_payload["write"]["write_performed"] is True
Expand Down
Loading
Loading