diff --git a/dtcli/scout.py b/dtcli/scout.py index 6760448..bde19c8 100644 --- a/dtcli/scout.py +++ b/dtcli/scout.py @@ -1,11 +1,12 @@ """Datatrail Scout Command.""" import logging -from typing import List +import re +from pathlib import PurePosixPath +from typing import Any, Dict, List, Optional import click import requests -from cadcutils.exceptions import BadRequestException from rich.console import Console from rich.prompt import Confirm from rich.table import Table @@ -25,6 +26,16 @@ console = Console() error_console = Console(stderr=True, style="bold red") +# Remote roots used by the Datatrail server's scout proxy. These are independent +# of the client's local root_mounts configuration. +SCOUT_ROOT_MOUNTS = { + "chime": "/", + "baseband_buffer": "/data/baseband_buffer/", + "kko": "/", + "gbo": "/", + "hco": "/", +} + @click.command(name="scout", help="Scout a dataset.") @click.argument("scopes", required=False, type=click.STRING, nargs=-1) @@ -63,14 +74,15 @@ def scout( # noqa: C901 if scopes: logger.debug(f"Scopes limited to: {list(scopes)}") try: - if not all([validate_scope(scope) for scope in scopes]): - error_console.print("A scope is invalid.") - console.print("Valid scopes are:") - ctx.invoke(ls) - return None - except Exception as e: - error_console.print(e) - return None + valid_scopes = all(validate_scope(scope) for scope in scopes) + except Exception: + error_console.print("Unable to validate scopes.") + ctx.exit(1) + if not valid_scopes: + error_console.print("A scope is invalid.") + console.print("Valid scopes are:") + ctx.invoke(ls) + ctx.exit(1) # Load configuration. try: @@ -81,72 +93,61 @@ def scout( # noqa: C901 logger.error( "No configuration file found. Create one with `datatrail config init`." ) - return {"error": "No config. Create one with `datatrail config init`."} + ctx.exit(1) # Check Canfar status. check_canfar_status(error_console) # Scout dataset. - endpoint = ( - f"/query/dataset/scout?name={dataset}" - if not scopes - else f"/query/dataset/scout?name={dataset}&{'&'.join([f'scopes={s}' for s in scopes])}" # noqa: E501 - ) - url = server + endpoint + url = server.rstrip("/") + "/query/dataset/scout" + params: Dict[str, Any] = {"name": dataset} + if scopes: + params["scopes"] = scopes logger.debug(f"URL: {url}") try: - response = requests.get(url, timeout=REQUEST_TIMEOUT) + response = requests.get(url, params=params, timeout=REQUEST_TIMEOUT) except requests.exceptions.Timeout: error_console.print("Error: Datatrail server timed out.") - return None + ctx.exit(1) + except requests.RequestException: + error_console.print("Error: Datatrail scout request failed.") + ctx.exit(1) + if not 200 <= response.status_code < 300: + error_console.print( + f"Error: Datatrail server returned HTTP {response.status_code}." + ) + ctx.exit(1) try: data = response.json() logger.debug(f"Data: {data}") - except requests.JSONDecodeError: - if "Response Timeout" in response.text: - error_console.print("Error: Datatrail server timed out.") - return None - else: - error_console.print(f"Error: {response.text}") - return None - - if "error" in data.keys(): - error_console.print(data["error"]) - return None + except ValueError: + error_console.print("Error: Datatrail server returned invalid JSON.") + ctx.exit(1) + if not _valid_scout_data(data): + error_console.print("Error: Datatrail returned no valid scout results.") + ctx.exit(1) - storage_elements = list(data[scopes[0]]["observed"].keys()) file_discrepancies: List[List] = [] + failed = False - for scope in data.keys(): - basepath = data.get(scope).get("basepath") + for scope in data: + basepath = data[scope]["basepath"].replace("'", "''") query = f"select count(*) from inventory.Artifact where uri like 'cadc:CHIMEFRB/{basepath}%'" # noqa: E501 try: - count, _ = cadcclient.query(query) - count = int(count[0]) - except BadRequestException as error: - error_console.print("Query failed.") - error_console.print(error) - return None - except Exception as error: - error_console.print("Query failed.") - error_console.print(error) - return None + count = int(cadcclient.query(query)[0][0]) + if count < 0: + raise ValueError("Invalid file count.") + except Exception: + error_console.print(f"{scope} - Minoc count query failed.") + count = -1 + failed = True data[scope]["observed"]["minoc"] = count - - keys_missing_in_observed = list( - set(data[scope]["expected"].keys()) - set(data[scope]["observed"].keys()) + storage_elements = dict.fromkeys( + [*data[scope]["observed"], *data[scope]["expected"]] ) - keys_missing_in_expected = list( - set(data[scope]["observed"].keys()) - set(data[scope]["expected"].keys()) - ) - - for key in keys_missing_in_observed: - data[scope]["observed"][key] = 0 - - for key in keys_missing_in_expected: - data[scope]["expected"][key] = 0 - for se in storage_elements: + data[scope]["observed"].setdefault(se, 0) + data[scope]["expected"].setdefault(se, 0) if data[scope]["observed"][se] > data[scope]["expected"][se]: file_discrepancies.append([scope, se]) @@ -156,39 +157,126 @@ def scout( # noqa: C901 error_console.print("File discrepancies:") for scope, se in file_discrepancies: error_console.print(f" - {se}: {scope}") - ifHeal = Confirm.ask("\nWould you like to attempt to heal this discrepancy?") - if ifHeal: - basepath = data.get(scope).get("basepath") - file_type = data.get(scope).get("filetype") - if se == "minoc": - file_md5s = cadcclient.dataset_md5s(basepath) - # console.print(minoc_md5s) - else: - md5_url = ( - server - + "/query/datasset/scout/md5sums" - + f"?basepath={basepath}&site={se}&filetype={file_type}" - ) - try: - response = requests.get(md5_url, timeout=REQUEST_TIMEOUT) - except requests.exceptions.Timeout: - error_console.print(f"{scope} - Healing timed out; skipping.") - continue - file_md5s = response.json() - url = ( - server - + "/commit/dataset/scout/sync" - + f"?name={dataset}&scope={scope}&replicate_to={se}" + if Confirm.ask("\nWould you like to attempt to heal this discrepancy?"): + if not _heal(server, dataset, scope, se, data[scope]): + failed = True + if failed: + ctx.exit(1) + + +def _valid_scout_data(data: Any) -> bool: + """Require the report fields used to display and select repairs.""" + if not isinstance(data, dict) or not data or "error" in data: + return False + for scope, info in data.items(): + if not isinstance(scope, str) or not scope or not isinstance(info, dict): + return False + if not isinstance(info.get("basepath"), str) or not info["basepath"]: + return False + if not isinstance(info.get("filetype"), str): + return False + for field in ("observed", "expected"): + counts = info.get(field) + if not isinstance(counts, dict) or not all( + isinstance(site, str) + and site + and isinstance(count, int) + and not isinstance(count, bool) + and count >= (0 if field == "expected" else -1) + for site, count in counts.items() + ): + return False + return True + + +def _validated_checksums(data: Any) -> Optional[Dict[str, str]]: + """Accept filename-to-MD5 mappings and normalize the optional MD5 prefix.""" + if not isinstance(data, dict) or not data or "error" in data: + return None + checksums = {} + for filename, value in data.items(): + if not isinstance(filename, str) or not filename.strip(): + return None + if not isinstance(value, str): + return None + checksum = value.strip().lower() + if checksum.startswith("md5:"): + checksum = checksum[4:] + if re.fullmatch(r"[0-9a-f]{32}", checksum) is None: + return None + checksums[filename] = checksum + return checksums + + +def _site_checksums( + checksums: Dict[str, str], site: str, basepath: str +) -> Optional[Dict[str, str]]: + """Convert scout filenames to the dataset-relative names used by sync.""" + if site not in SCOUT_ROOT_MOUNTS: + return None + root = PurePosixPath(SCOUT_ROOT_MOUNTS[site]) + base = PurePosixPath(basepath) + if base.is_absolute() or ".." in base.parts: + return None + normalized = {} + for filename, checksum in checksums.items(): + path = PurePosixPath(filename) + if ".." in path.parts: + return None + try: + relative = path.relative_to(root) if path.is_absolute() else path + suffix = relative.relative_to(base) + except ValueError: + return None + name = relative.as_posix() + if not suffix.parts or name in normalized: + return None + normalized[name] = checksum + return normalized + + +def _heal(server: str, dataset: str, scope: str, site: str, info: Dict) -> bool: + """Fetch validated checksums and submit one explicitly confirmed repair.""" + try: + if site == "minoc": + data = cadcclient.dataset_md5s(info["basepath"]) + else: + response = requests.get( + server.rstrip("/") + "/query/dataset/scout/md5sums", + params={ + "basepath": info["basepath"], + "site": site, + "filetype": info["filetype"], + }, + timeout=REQUEST_TIMEOUT, ) - try: - response = requests.post(url, json=file_md5s, timeout=REQUEST_TIMEOUT) - except requests.exceptions.Timeout: - error_console.print(f"{scope} - Healing timed out; skipping.") - continue - if response.status_code == 200: - console.print(f"{scope} - Healing successful.") - else: - error_console.print(f"{scope} - Healing failed.") + if not 200 <= response.status_code < 300: + raise ValueError("Checksum request failed.") + data = response.json() + except Exception: + error_console.print(f"{scope} ({site}) - Checksum query failed; skipping.") + return False + checksums = _validated_checksums(data) + if checksums is not None and site != "minoc": + checksums = _site_checksums(checksums, site, info["basepath"]) + if checksums is None: + error_console.print(f"{scope} ({site}) - Invalid checksum response; skipping.") + return False + try: + response = requests.post( + server.rstrip("/") + "/commit/dataset/scout/sync", + params={"name": dataset, "scope": scope, "replicate_to": site}, + json=checksums, + timeout=REQUEST_TIMEOUT, + ) + except requests.RequestException: + error_console.print(f"{scope} ({site}) - Healing failed.") + return False + if not 200 <= response.status_code < 300: + error_console.print(f"{scope} ({site}) - Healing failed.") + return False + console.print(f"{scope} ({site}) - Healing successful.") + return True def show_scout_results(dataset: str, data: dict): @@ -200,7 +288,9 @@ def show_scout_results(dataset: str, data: dict): """ # Display results. scopes = list(data.keys()) - storage_elements = list(data[scopes[0]]["observed"].keys()) + storage_elements = dict.fromkeys( + site for info in data.values() for site in [*info["observed"], *info["expected"]] + ) table = Table( title=f"Scout Results for {dataset}", header_style="magenta", @@ -214,13 +304,13 @@ def show_scout_results(dataset: str, data: dict): # Observed row = [scope] for se in storage_elements: - row.append(str(data[scope]["observed"][se])) + row.append(str(data[scope]["observed"].get(se, 0))) table.add_row(*row, style="blue") # Expected row = [scope] for se in storage_elements: - row.append(str(data[scope]["expected"][se])) + row.append(str(data[scope]["expected"].get(se, 0))) table.add_row(*row, style="yellow", end_section=True) console.print(table) diff --git a/tests/test_scout.py b/tests/test_scout.py new file mode 100644 index 0000000..be0f9e6 --- /dev/null +++ b/tests/test_scout.py @@ -0,0 +1,419 @@ +"""Offline regressions for scout reports and confirmed healing requests.""" + +import json +from types import SimpleNamespace +from typing import Any, List +from unittest.mock import Mock + +import pytest +import requests +from click.testing import CliRunner + +from dtcli import scout + +INVALID_REPORTS: List[Any] = [None, [], {}, {"error": "private-error"}, {"scope": {}}] + + +def response(payload=None, status=200, text=None): + """Create a real Requests response, including its normal JSON/status checks.""" + result = requests.Response() + result.status_code = status + result._content = (json.dumps(payload) if text is None else text).encode() + return result + + +def record(observed=None, expected=None): + """Match the server response, whose observed sites do not include Minoc.""" + return { + "basepath": "data/event/1", + "filetype": ".h5", + "observed": {"chime": 1} if observed is None else observed, + "expected": {"chime": 1, "minoc": 0} if expected is None else expected, + } + + +@pytest.fixture +def services(monkeypatch): + """Replace every service and prompt without reading the user's configuration.""" + mocked = SimpleNamespace( + get=Mock(return_value=response({"scope": record()})), + post=Mock(return_value=response("ran successfully")), + query=Mock(return_value=[["0"], [""]]), + checksums=Mock(return_value={"data/event/1/file.h5": "a" * 32}), + confirm=Mock(return_value=False), + config={ + "server": "https://example.invalid/datatrail", + "root_mounts": {"chime": "/", "hco": "/"}, + }, + ) + monkeypatch.setattr(scout, "procure", lambda: mocked.config) + monkeypatch.setattr(scout, "validate_scope", lambda scope: True) + monkeypatch.setattr(scout, "check_canfar_status", lambda console: (True, True)) + monkeypatch.setattr(scout.requests, "get", mocked.get) + monkeypatch.setattr(scout.requests, "post", mocked.post) + monkeypatch.setattr(scout.cadcclient, "query", mocked.query) + monkeypatch.setattr(scout.cadcclient, "dataset_md5s", mocked.checksums) + monkeypatch.setattr(scout.Confirm, "ask", mocked.confirm) + return mocked + + +@pytest.mark.parametrize("arguments", [["dataset"], ["scope", "dataset"]]) +def test_scout_accepts_optional_scopes(services, arguments): + """Use returned scopes for both documented invocation forms.""" + result = CliRunner().invoke(scout.scout, arguments) + + assert result.exit_code == 0, result.output + assert "scope" in result.output + assert "minoc" in result.output + services.post.assert_not_called() + + +@pytest.mark.parametrize("checksum", ["a" * 32, "md5:" + "A" * 32]) +def test_scout_heals_minoc_missing_from_server_observed(services, checksum): + """Include the separately queried Minoc count when selecting discrepancies.""" + services.query.return_value = [["1"], [""]] + services.checksums.return_value = {"data/event/1/file.h5": checksum} + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 0, result.output + services.confirm.assert_called_once() + services.checksums.assert_called_once_with("data/event/1") + services.post.assert_called_once_with( + "https://example.invalid/datatrail/commit/dataset/scout/sync", + params={"name": "dataset", "scope": "scope", "replicate_to": "minoc"}, + json={"data/event/1/file.h5": "a" * 32}, + timeout=scout.REQUEST_TIMEOUT, + ) + + +def test_scout_renders_and_checks_different_site_sets(services): + """Each scope has its own discrepancies and all sites appear in the table.""" + services.get.return_value = response( + { + "first": record({"chime": 2}, {"chime": 1}), + "second": record({"baseband_buffer": 2}, {"baseband_buffer": 1, "hco": 1}), + } + ) + + result = CliRunner().invoke(scout.scout, ["first", "second", "dataset"]) + + assert result.exit_code == 0, result.output + assert "chime: first" in result.output + assert "baseband_buffer: second" in result.output + assert "hco" in result.output + assert services.confirm.call_count == 2 + services.post.assert_not_called() + + +def test_scout_declined_healing_never_fetches_or_posts_checksums(services): + """Leave the archive unchanged unless the specific repair is confirmed.""" + services.query.return_value = [["1"]] + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 0, result.output + services.confirm.assert_called_once() + services.checksums.assert_not_called() + services.post.assert_not_called() + + +@pytest.mark.parametrize( + "site, root", [("baseband_buffer", "/data/baseband_buffer/"), ("chime", "/")] +) +def test_scout_non_minoc_healing_uses_correct_endpoint_and_payload(services, site, root): + """Fetch site checksums from the real route and submit the validated mapping.""" + checksums = {"data/event/1/file.h5": "0123456789abcdef" * 2} + services.get.side_effect = [ + response({"scope": record({site: 1}, {site: 0})}), + response({root + filename: value for filename, value in checksums.items()}), + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 0, result.output + assert services.get.call_args_list[1].args == ( + "https://example.invalid/datatrail/query/dataset/scout/md5sums", + ) + assert services.get.call_args_list[1].kwargs == { + "params": { + "basepath": "data/event/1", + "site": site, + "filetype": ".h5", + }, + "timeout": scout.REQUEST_TIMEOUT, + } + assert services.post.call_args.kwargs["json"] == checksums + assert services.post.call_args.kwargs["params"]["replicate_to"] == site + + +def test_scout_remote_root_is_independent_of_client_mount(services): + """Client filesystem mounts cannot change the remote scout path contract.""" + services.config["root_mounts"]["baseband_buffer"] = "/archive/baseband/" + services.get.side_effect = [ + response({"scope": record({"baseband_buffer": 1}, {"baseband_buffer": 0})}), + response({"/data/baseband_buffer/data/event/1/file.h5": "a" * 32}), + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 0, result.output + assert services.post.call_args.kwargs["json"] == {"data/event/1/file.h5": "a" * 32} + + +def test_scout_heals_baseband_buffer_and_chime_independently(services): + """Repair discrepancies at both scout sites even when Minoc has no files.""" + services.get.side_effect = [ + response({"scope": record({"baseband_buffer": 2, "chime": 1}, {})}), + response( + { + "/data/baseband_buffer/data/event/1/first.h5": "a" * 32, + "/data/baseband_buffer/data/event/1/second.h5": "b" * 32, + } + ), + response({"/data/event/1/first.h5": "a" * 32}), + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 0, result.output + assert services.confirm.call_count == 2 + assert services.post.call_count == 2 + calls = services.post.call_args_list + assert calls[0].kwargs["params"]["replicate_to"] == "baseband_buffer" + assert calls[0].kwargs["json"] == { + "data/event/1/first.h5": "a" * 32, + "data/event/1/second.h5": "b" * 32, + } + assert calls[1].kwargs["params"]["replicate_to"] == "chime" + assert calls[1].kwargs["json"] == {"data/event/1/first.h5": "a" * 32} + services.checksums.assert_not_called() + + +@pytest.mark.parametrize( + "filenames", + [ + ["/archive/baseband/data/event/1/file.h5"], + ["/data/baseband_buffer_other/data/event/1/file.h5"], + ["/data/baseband_buffer/data/event/10/file.h5"], + ["/data/baseband_buffer/data/event/1/../1/file.h5"], + ["/data/baseband_buffer/data/event/1"], + ["/data/baseband_buffer/data/event/1/file.h5", "data/event/1/file.h5"], + ["data/event/10/file.h5"], + ], +) +def test_scout_rejects_checksums_outside_root_or_dataset(services, filenames): + """Never guess a path prefix or submit ambiguous names for a confirmed repair.""" + services.get.side_effect = [ + response({"scope": record({"baseband_buffer": 1}, {"baseband_buffer": 0})}), + response(dict.fromkeys(filenames, "a" * 32)), + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + assert "Invalid checksum response" in result.output + services.post.assert_not_called() + + +def test_scout_unknown_remote_site_prevents_healing(services): + """Unknown remote mounts must fail rather than infer a prefix from filenames.""" + services.get.side_effect = [ + response({"scope": record({"new_site": 1}, {"new_site": 0})}), + response({"/new_mount/data/event/1/file.h5": "a" * 32}), + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + services.post.assert_not_called() + + +def test_scout_negative_expected_count_prevents_healing(services): + """An invalid expected count must not make a missing replica look repairable.""" + services.get.return_value = response({"scope": record({"chime": 0}, {"chime": -1})}) + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + services.confirm.assert_not_called() + services.post.assert_not_called() + + +def test_scout_unknown_observed_count_does_not_prompt_healing(services): + """The server's -1 sentinel represents an unavailable scout, not a replica.""" + services.get.return_value = response({"scope": record({"chime": -1}, {"chime": 0})}) + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 0, result.output + services.confirm.assert_not_called() + services.post.assert_not_called() + + +@pytest.mark.parametrize("error", [requests.ConnectionError, requests.Timeout]) +def test_scout_request_errors_are_concise_failures(services, error): + """Report expected transport failures without an uncaught exception.""" + services.get.side_effect = error("private-request-details") + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + assert "failed" in result.output.lower() or "timed out" in result.output.lower() + assert "private-request-details" not in result.output + services.post.assert_not_called() + + +@pytest.mark.parametrize("payload", INVALID_REPORTS) +def test_scout_rejects_empty_or_malformed_results(services, payload): + """Avoid table/index errors and never heal from an invalid report.""" + services.get.return_value = response(payload) + + result = CliRunner().invoke(scout.scout, ["dataset"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + services.query.assert_not_called() + services.confirm.assert_not_called() + services.post.assert_not_called() + + +def test_scout_rejects_failed_http_status_before_using_payload(services): + """A valid-looking JSON document cannot override HTTP failure.""" + services.get.return_value = response({"scope": record()}, status=503) + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + services.query.assert_not_called() + services.post.assert_not_called() + + +@pytest.mark.parametrize( + "payload", + [None, [], {}, {"error": "failed"}, {"file": ""}, {"file": 1}, {"file": "not-md5"}], +) +def test_scout_invalid_checksum_payload_prevents_healing(services, payload): + """Never submit empty, error, or malformed checksum responses as replicas.""" + services.get.side_effect = [ + response({"scope": record({"chime": 2}, {"chime": 1})}), + response(payload), + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + services.post.assert_not_called() + + +@pytest.mark.parametrize( + "failure", + [ + requests.ConnectionError("private"), + requests.Timeout("private"), + response({}, 503), + response(text="not-json"), + ], +) +def test_scout_checksum_get_failure_prevents_post(services, failure): + """Failed checksum requests cannot trigger a healing POST.""" + services.get.side_effect = [ + response({"scope": record({"chime": 2}, {"chime": 1})}), + failure, + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + assert "private" not in result.output + services.post.assert_not_called() + + +def test_scout_continues_independent_healing_after_failure(services): + """A failed repair leaves the command unsuccessful but permits another repair.""" + services.get.side_effect = [ + response({"scope": record({"chime": 2, "hco": 2}, {"chime": 1, "hco": 1})}), + response({}, status=503), + response({"data/event/1/file.h5": "a" * 32}), + ] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + assert services.confirm.call_count == 2 + services.post.assert_called_once() + assert services.post.call_args.kwargs["params"]["replicate_to"] == "hco" + assert "Healing successful" in result.output + + +@pytest.mark.parametrize( + "failure", [requests.ConnectionError("private"), response({}, 503)] +) +def test_scout_failed_healing_post_is_nonzero(services, failure): + """Report failed writes as command failures after explicit confirmation.""" + services.query.return_value = [["1"], [""]] + services.confirm.return_value = True + services.post.side_effect = [failure] + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + assert "Healing failed" in result.output + assert "private" not in result.output + + +def test_scout_invalid_initial_json_is_concise(services): + """Do not expose malformed server contents or proceed to healing.""" + services.get.return_value = response(text="private-response") + + result = CliRunner().invoke(scout.scout, ["dataset"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, SystemExit) + assert "invalid JSON" in result.output + assert "private-response" not in result.output + services.post.assert_not_called() + + +def test_scout_count_failure_does_not_heal_unknown_minoc_state(services): + """Continue checking independent scopes without repairing an unknown count.""" + services.get.return_value = response({"first": record(), "second": record()}) + services.query.side_effect = [RuntimeError("private-query"), [["1"]]] + services.confirm.return_value = True + + result = CliRunner().invoke(scout.scout, ["dataset"]) + + assert result.exit_code == 1 + services.confirm.assert_called_once() + services.post.assert_called_once() + assert services.post.call_args.kwargs["params"]["scope"] == "second" + assert "private-query" not in result.output + + +def test_scout_minoc_checksum_failure_prevents_healing(services): + """A failed inventory checksum lookup cannot be submitted as a repair.""" + services.query.return_value = [["1"]] + services.confirm.return_value = True + services.checksums.side_effect = RuntimeError("private-query") + + result = CliRunner().invoke(scout.scout, ["scope", "dataset"]) + + assert result.exit_code == 1 + assert "Checksum query failed" in result.output + assert "private-query" not in result.output + services.post.assert_not_called()