diff --git a/.github/actions/contributor-check/tests/test_probe_search_unavailable.py b/.github/actions/contributor-check/tests/test_probe_search_unavailable.py new file mode 100644 index 0000000..d695e51 --- /dev/null +++ b/.github/actions/contributor-check/tests/test_probe_search_unavailable.py @@ -0,0 +1,85 @@ +"""The credential and cluster probes on an account GitHub search refuses. + +Both probes read ``author:`` searches. Their ``_api`` turned every 422 into +``None``, so a refused search read as an empty history and scored NONE, the one +reading that looks clean. A refusal now reports UNKNOWN; any other 422 keeps +its old meaning. Follow-up to agentrust-io/.github#56, raised on #57. +""" +from __future__ import annotations + +import importlib.util +import io +import json +import sys +from pathlib import Path +from urllib.error import HTTPError + +import pytest + +VENDOR = Path(__file__).resolve().parents[1] / "vendor" / "agt" + +REFUSED = ( + b'{"message":"Validation Failed","errors":[{"message":"The listed users cannot be ' + b'searched either because the users do not exist or you do not have permission to ' + b'view the users.","resource":"Search","field":"q","code":"invalid"}]}' +) +OTHER_422 = b'{"message":"Validation Failed"}' + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(f"agt_{name}", VENDOR / f"{name}.py") + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + + +class _Resp: + def __init__(self, payload): + self._payload = json.dumps(payload).encode() + + def read(self): + return self._payload + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + +def _fake_urlopen(search_body: bytes): + def fake(req, timeout=None): + url = req.full_url + if "/search/" in url: + raise HTTPError(url, 422, "err", {}, io.BytesIO(search_body)) + if url.rstrip("/").endswith("/users/someone"): + return _Resp({"login": "someone", "created_at": "2014-05-17T06:30:28Z", + "public_repos": 0, "followers": 0, "following": 0}) + return _Resp([]) + return fake + + +@pytest.fixture(params=["credential_audit", "cluster_detect"]) +def probe(request, monkeypatch): + mod = _load(request.param) + monkeypatch.setattr(mod, "_get_token", lambda: "t") + return request.param, mod + + +def _risk(name, mod): + if name == "credential_audit": + return mod.audit_credentials("someone", "agentrust-io/.github").risk + return mod.detect_cluster("someone").risk_level() + + +def test_refused_search_is_unknown_not_clean(probe, monkeypatch): + name, mod = probe + monkeypatch.setattr(mod, "urlopen", _fake_urlopen(REFUSED)) + assert _risk(name, mod) == "UNKNOWN" + + +def test_other_422_keeps_its_old_reading(probe, monkeypatch): + name, mod = probe + monkeypatch.setattr(mod, "urlopen", _fake_urlopen(OTHER_422)) + assert _risk(name, mod) == "NONE" diff --git a/.github/actions/contributor-check/tests/test_search_unavailable.py b/.github/actions/contributor-check/tests/test_search_unavailable.py new file mode 100644 index 0000000..01babb3 --- /dev/null +++ b/.github/actions/contributor-check/tests/test_search_unavailable.py @@ -0,0 +1,75 @@ +"""GitHub search refusing an account's ``author:`` queries (agentrust-io/.github#56). + +The search API answers 422 "The listed users cannot be searched" for some +accounts with public profiles. That is the API declining to answer, so the +check reports the account as undetermined instead of crashing on it. +""" +from __future__ import annotations + +import io +from urllib.error import HTTPError + +import pytest + +from test_api_retry import _http, _load, _seq + +REFUSED = ( + b'{"message":"Validation Failed","errors":[{"message":"The listed users cannot be ' + b'searched either because the users do not exist or you do not have permission to ' + b'view the users.","resource":"Search","field":"q","code":"invalid"}]}' +) + + +def _http_body(code: int, body: bytes) -> HTTPError: + return HTTPError("https://api.github.com/search/issues", code, "err", {}, io.BytesIO(body)) + + +@pytest.fixture() +def cc(monkeypatch): + mod = _load() + monkeypatch.setattr(mod, "_get_token", lambda: "t") + monkeypatch.setattr(mod.time, "sleep", lambda _s: None) + return mod + + +def test_refused_author_search_raises_search_unavailable(cc, monkeypatch): + _seq(monkeypatch, cc, [_http_body(422, REFUSED)]) + with pytest.raises(cc.SearchUnavailable): + cc._search_issues("author:someone is:issue") + + +def test_other_422_is_still_reported_as_itself(cc, monkeypatch): + _seq(monkeypatch, cc, [_http_body(422, b'{"message":"Validation Failed"}')]) + with pytest.raises(HTTPError) as info: + cc._search_issues("author:someone is:issue") + assert not isinstance(info.value, cc.SearchUnavailable) + + +def test_unretried_status_names_the_request_path(cc, monkeypatch, capsys): + _seq(monkeypatch, cc, [_http(422)]) + with pytest.raises(HTTPError): + cc._api("/search/issues", {"q": "author:someone"}) + last = capsys.readouterr().err.strip().splitlines()[-1] + assert "422" in last and "/search/issues" in last, last + + +def test_exhausted_server_error_names_the_request_path(cc, monkeypatch, capsys): + _seq(monkeypatch, cc, [_http(503)]) + with pytest.raises(HTTPError): + cc._api("/users/someone/repos") + assert "/users/someone/repos" in capsys.readouterr().err.strip().splitlines()[-1] + + +def test_account_hidden_from_search_is_unknown_not_scored(cc, monkeypatch): + def fake_api(path, params=None): + if path == "/users/someone": + return {"login": "someone", "created_at": "2014-05-17T06:30:28Z", + "public_repos": 3, "followers": 0, "following": 0} + if path == "/search/issues": + raise _http_body(422, REFUSED) + return [] + + monkeypatch.setattr(cc, "_api", fake_api) + report = cc.check_contributor("someone", "agentrust-io/.github") + assert report.risk == "UNKNOWN" + assert [s.name for s in report.signals if s.name == "search_unavailable"] == ["search_unavailable"] diff --git a/.github/actions/contributor-check/vendor/agt/README.md b/.github/actions/contributor-check/vendor/agt/README.md index a044d14..9d318f0 100644 --- a/.github/actions/contributor-check/vendor/agt/README.md +++ b/.github/actions/contributor-check/vendor/agt/README.md @@ -18,15 +18,17 @@ level. See agentrust-io/.github#27. |---|---|---| | `contributor_check.py` | `scripts/contributor_check.py` | modified, see below | | `contributor_check_allowlist.json` | `scripts/contributor_check_allowlist.json` | `d8a5ddbb0695c21b3eae88e6bc85edcda6cbd9721cad3ca9d10fbeeed4fc0636` | -| `credential_audit.py` | `scripts/credential_audit.py` | `9650675038e7720a6d5cdd8be058fa38b2e004cc1e6f7bc81c48ed100e4680e1` | -| `cluster_detect.py` | `scripts/cluster_detect.py` | `da36c10074aa638bf61c5873fbe6cff975533563553689eba5e90329e3d8d7cf` | +| `credential_audit.py` | `scripts/credential_audit.py` | modified, see below | +| `cluster_detect.py` | `scripts/cluster_detect.py` | modified, see below | | `contributor_check_action.py` | `scripts/contributor_check_action.py` | modified, see below | -The first three unmodified files are byte-identical to upstream at that commit. -The checksums above are the upstream bytes, so `sha256sum` here is a one-line -verification for those three. +The unmodified file is byte-identical to upstream at that commit. Its checksum +above is the upstream bytes, so `sha256sum` here is a one-line verification. +`credential_audit.py` was `9650675038e7720a6d5cdd8be058fa38b2e004cc1e6f7bc81c48ed100e4680e1` +and `cluster_detect.py` was `da36c10074aa638bf61c5873fbe6cff975533563553689eba5e90329e3d8d7cf` +upstream before the change below. -## The two local modifications +## Local modifications ### `contributor_check_action.py` @@ -57,7 +59,24 @@ The retry is limited to transient failures and reports anything else as itself: | 403 rate limit | Honours `Retry-After`, clamped to 5-60s, within the attempt bound. | | 5xx | Full-jitter exponential backoff within the attempt bound. | | `URLError` | Full-jitter exponential backoff within the attempt bound. | -| Any other HTTP status | Raised on the first attempt. | +| Any other HTTP status | Raised on the first attempt, after a stderr line naming the status and the request path. | + +Every exhausted-retry line names the request path as well, since the action +surfaces only the last stderr line. + +`_search_issues` raises `SearchUnavailable` when the search API answers 422 +"The listed users cannot be searched", which it does for some accounts with +public profiles under any token but their own. `check_contributor` then reports +`UNKNOWN` with a `search_unavailable` signal after the account-shape and +repo-theme checks, rather than crashing or scoring only the checks that could +run. See agentrust-io/.github#56. + +### `credential_audit.py` and `cluster_detect.py` + +`credential_audit.py` and `cluster_detect.py` recognise the same refusal in +their own `_api` and report `UNKNOWN` for it. Before this, their catch-all for +404 and 422 read a refused search as an empty history, which scored `NONE`. +Any other 422 is still read as before. When a retryable condition outlasts the bound, the final line written to stderr names the cause before the exception propagates. `contributor_check_action.py` @@ -85,6 +104,6 @@ here. The commit above is provenance, not a sync point, and no resync is planned. Change these files directly, as you would any other file in this repository. -If a change lands in one of the three that remain byte-identical, move it out of -the checksum table above and out of the `vendor-integrity` job, and describe the -change here, the way the two modified files are described. +If a change lands in the file that remains byte-identical, move it out of the +checksum table above and out of the `vendor-integrity` job, and describe the +change here, the way the modified files are described. diff --git a/.github/actions/contributor-check/vendor/agt/cluster_detect.py b/.github/actions/contributor-check/vendor/agt/cluster_detect.py index b250175..70338ec 100644 --- a/.github/actions/contributor-check/vendor/agt/cluster_detect.py +++ b/.github/actions/contributor-check/vendor/agt/cluster_detect.py @@ -57,6 +57,22 @@ def _get_token() -> str: return token +class SearchUnavailable(Exception): + """GitHub search refuses ``author:`` queries for this account. + + Same refusal as in ``contributor_check.py``. Returning ``None`` here would + read as an empty history and score clean. See agentrust-io/.github#56. + """ + + +def _is_search_refusal(exc: HTTPError) -> bool: + try: + body = exc.read().decode("utf-8", "replace") + except Exception: + return False + return "cannot be searched" in body + + def _api(path: str, params: dict[str, str] | None = None) -> Any: url = f"https://api.github.com{path}" if params: @@ -77,6 +93,8 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: print(f" Rate limited, waiting {wait}s...", file=sys.stderr) import time; time.sleep(wait) continue + if exc.code == 422 and _is_search_refusal(exc): + raise SearchUnavailable(path) from exc if exc.code in (404, 422): return None raise @@ -119,6 +137,7 @@ class ClusterReport: accounts: dict[str, AccountInfo] = field(default_factory=dict) edges: list[Edge] = field(default_factory=list) shared_forks: dict[str, list[str]] = field(default_factory=dict) + search_unavailable: bool = False @property def account_count(self) -> int: @@ -129,6 +148,8 @@ def edge_count(self) -> int: return len(self.edges) def risk_level(self) -> str: + if self.search_unavailable: + return "UNKNOWN" if self.account_count >= 5 and self.edge_count >= 8: return "HIGH" elif self.account_count >= 3 and self.edge_count >= 4: @@ -327,6 +348,16 @@ def detect_cluster(seed: str, depth: int = 1) -> ClusterReport: return report report.accounts[seed] = seed_info + try: + _walk(report, seed, depth) + except SearchUnavailable: + # The co-comment and sync-filing passes read author: search. Without + # them the edge count is not established, so no level is reported. + report.search_unavailable = True + return report + + +def _walk(report: ClusterReport, seed: str, depth: int) -> None: visited = {seed} frontier = {seed} @@ -377,8 +408,6 @@ def detect_cluster(seed: str, depth: int = 1) -> ClusterReport: unique_edges.append(edge) report.edges = unique_edges - return report - # --------------------------------------------------------------------------- # Output formatting diff --git a/.github/actions/contributor-check/vendor/agt/contributor_check.py b/.github/actions/contributor-check/vendor/agt/contributor_check.py index 9c8ace2..7e13f2b 100644 --- a/.github/actions/contributor-check/vendor/agt/contributor_check.py +++ b/.github/actions/contributor-check/vendor/agt/contributor_check.py @@ -116,7 +116,7 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: continue print( f" Rate limited by the GitHub API and still limited after " - f"{_RETRY_MAX_ATTEMPTS} attempts", + f"{_RETRY_MAX_ATTEMPTS} attempts on {path}", file=sys.stderr, ) raise @@ -132,11 +132,13 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: continue print( f" GitHub API returned {exc.code} on all " - f"{_RETRY_MAX_ATTEMPTS} attempts", + f"{_RETRY_MAX_ATTEMPTS} attempts on {path}", file=sys.stderr, ) raise - # Anything else is the API answering. Report it as itself. + # Anything else is the API answering. Report it as itself, and + # name the call: the action surfaces only the last stderr line. + print(f" GitHub API returned {exc.code} on {path}", file=sys.stderr) raise except URLError as exc: # Network-layer failures: DNS, TCP reset, TLS abort, socket @@ -152,15 +154,35 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: continue print( f" Network error ({exc.reason}) on all " - f"{_RETRY_MAX_ATTEMPTS} attempts", + f"{_RETRY_MAX_ATTEMPTS} attempts on {path}", file=sys.stderr, ) raise +class SearchUnavailable(Exception): + """GitHub search refuses to run an ``author:`` query for this account. + + The search API answers 422 "The listed users cannot be searched" for some + accounts whose profiles are otherwise public, under any token other than + the account's own. No issue history can be read for them, so the signals + built on it are not established. See agentrust-io/.github#56. + """ + + def _search_issues(query: str, per_page: int = 30) -> list[dict]: """Search GitHub issues/PRs.""" - data = _api("/search/issues", {"q": query, "per_page": str(per_page)}) + try: + data = _api("/search/issues", {"q": query, "per_page": str(per_page)}) + except HTTPError as exc: + if exc.code == 422: + try: + body = exc.read().decode("utf-8", "replace") + except Exception: + body = "" + if "cannot be searched" in body: + raise SearchUnavailable(query) from exc + raise return data.get("items", []) if data else [] @@ -1054,7 +1076,10 @@ def check_contributor(username: str, target_repo: str | None = None) -> Reputati # Shared data fetches (avoids redundant API calls across checkers) repos = _api(f"/users/{username}/repos", {"per_page": "100", "sort": "created"}) or [] - issues = _search_issues(f"author:{username} is:issue", per_page=100) + try: + issues = _search_issues(f"author:{username} is:issue", per_page=100) + except SearchUnavailable: + issues = None # Run checks with shared data for signal in check_account_shape(user): @@ -1063,6 +1088,21 @@ def check_contributor(username: str, target_repo: str | None = None) -> Reputati for signal in check_repo_themes(username, repos=repos): report.add(signal) + if issues is None: + # Every remaining check reads issue or PR search. Report the account + # as undetermined rather than scoring the checks that could run as if + # they were all of them. + report.add(Signal( + name="search_unavailable", + severity="LOW", + detail=( + "GitHub search refuses author: queries for this account, so " + "spray, credibility, credential and overlap signals were not established" + ), + )) + report.risk = "UNKNOWN" + return report + for signal in check_spray_pattern(username, issues=issues, user_repos=repos): report.add(signal) diff --git a/.github/actions/contributor-check/vendor/agt/credential_audit.py b/.github/actions/contributor-check/vendor/agt/credential_audit.py index 680b804..1a42509 100644 --- a/.github/actions/contributor-check/vendor/agt/credential_audit.py +++ b/.github/actions/contributor-check/vendor/agt/credential_audit.py @@ -59,6 +59,22 @@ def _get_token() -> str: return token +class SearchUnavailable(Exception): + """GitHub search refuses ``author:`` queries for this account. + + Same refusal as in ``contributor_check.py``. Returning ``None`` here would + read as an empty history and score clean. See agentrust-io/.github#56. + """ + + +def _is_search_refusal(exc: HTTPError) -> bool: + try: + body = exc.read().decode("utf-8", "replace") + except Exception: + return False + return "cannot be searched" in body + + def _api(path: str, params: dict[str, str] | None = None) -> Any: url = f"https://api.github.com{path}" if params: @@ -81,6 +97,8 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: print(f" Rate limited, waiting {wait}s...", file=sys.stderr) import time; time.sleep(wait) continue + if exc.code == 422 and _is_search_refusal(exc): + raise SearchUnavailable(path) from exc if exc.code in (404, 422): return None raise @@ -285,7 +303,12 @@ def audit_credentials(username: str, target_repo: str) -> CredentialAuditReport: report = CredentialAuditReport(username=username, target_repo=target_repo) # Step 1: Find merges - report.merges = find_merges(username, target_repo) + try: + report.merges = find_merges(username, target_repo) + except SearchUnavailable: + # No merge history can be read, which is not the same as none. + report.risk = "UNKNOWN" + return report if not report.merges: report.risk = "NONE" return report diff --git a/.github/workflows/contributor-check-tests.yml b/.github/workflows/contributor-check-tests.yml index 34019f1..fe3e959 100644 --- a/.github/workflows/contributor-check-tests.yml +++ b/.github/workflows/contributor-check-tests.yml @@ -39,7 +39,5 @@ jobs: run: | cd .github/actions/contributor-check/vendor/agt sha256sum -c --strict - <<'SUMS' - 9650675038e7720a6d5cdd8be058fa38b2e004cc1e6f7bc81c48ed100e4680e1 credential_audit.py - da36c10074aa638bf61c5873fbe6cff975533563553689eba5e90329e3d8d7cf cluster_detect.py d8a5ddbb0695c21b3eae88e6bc85edcda6cbd9721cad3ca9d10fbeeed4fc0636 contributor_check_allowlist.json SUMS