From 66dc240236168a71c3d7a148ae2866bdf94384a0 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Mon, 5 Oct 2026 08:31:54 -0700 Subject: [PATCH 1/2] fix(contributor-check): report an account GitHub search refuses as UNKNOWN The search API answers 422 "The listed users cannot be searched" for some accounts, which crashed the check. Report those as UNKNOWN with a search_unavailable signal, and name the request path on every API failure line. Closes #56. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Imran Siddique --- .../tests/test_search_unavailable.py | 75 +++++++++++++++++++ .../contributor-check/vendor/agt/README.md | 12 ++- .../vendor/agt/contributor_check.py | 52 +++++++++++-- 3 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 .github/actions/contributor-check/tests/test_search_unavailable.py diff --git a/.github/actions/contributor-check/tests/test_search_unavailable.py b/.github/actions/contributor-check/tests/test_search_unavailable.py new file mode 100644 index 0000000..01babb3 --- /dev/null +++ b/.github/actions/contributor-check/tests/test_search_unavailable.py @@ -0,0 +1,75 @@ +"""GitHub search refusing an account's ``author:`` queries (agentrust-io/.github#56). + +The search API answers 422 "The listed users cannot be searched" for some +accounts with public profiles. That is the API declining to answer, so the +check reports the account as undetermined instead of crashing on it. +""" +from __future__ import annotations + +import io +from urllib.error import HTTPError + +import pytest + +from test_api_retry import _http, _load, _seq + +REFUSED = ( + b'{"message":"Validation Failed","errors":[{"message":"The listed users cannot be ' + b'searched either because the users do not exist or you do not have permission to ' + b'view the users.","resource":"Search","field":"q","code":"invalid"}]}' +) + + +def _http_body(code: int, body: bytes) -> HTTPError: + return HTTPError("https://api.github.com/search/issues", code, "err", {}, io.BytesIO(body)) + + +@pytest.fixture() +def cc(monkeypatch): + mod = _load() + monkeypatch.setattr(mod, "_get_token", lambda: "t") + monkeypatch.setattr(mod.time, "sleep", lambda _s: None) + return mod + + +def test_refused_author_search_raises_search_unavailable(cc, monkeypatch): + _seq(monkeypatch, cc, [_http_body(422, REFUSED)]) + with pytest.raises(cc.SearchUnavailable): + cc._search_issues("author:someone is:issue") + + +def test_other_422_is_still_reported_as_itself(cc, monkeypatch): + _seq(monkeypatch, cc, [_http_body(422, b'{"message":"Validation Failed"}')]) + with pytest.raises(HTTPError) as info: + cc._search_issues("author:someone is:issue") + assert not isinstance(info.value, cc.SearchUnavailable) + + +def test_unretried_status_names_the_request_path(cc, monkeypatch, capsys): + _seq(monkeypatch, cc, [_http(422)]) + with pytest.raises(HTTPError): + cc._api("/search/issues", {"q": "author:someone"}) + last = capsys.readouterr().err.strip().splitlines()[-1] + assert "422" in last and "/search/issues" in last, last + + +def test_exhausted_server_error_names_the_request_path(cc, monkeypatch, capsys): + _seq(monkeypatch, cc, [_http(503)]) + with pytest.raises(HTTPError): + cc._api("/users/someone/repos") + assert "/users/someone/repos" in capsys.readouterr().err.strip().splitlines()[-1] + + +def test_account_hidden_from_search_is_unknown_not_scored(cc, monkeypatch): + def fake_api(path, params=None): + if path == "/users/someone": + return {"login": "someone", "created_at": "2014-05-17T06:30:28Z", + "public_repos": 3, "followers": 0, "following": 0} + if path == "/search/issues": + raise _http_body(422, REFUSED) + return [] + + monkeypatch.setattr(cc, "_api", fake_api) + report = cc.check_contributor("someone", "agentrust-io/.github") + assert report.risk == "UNKNOWN" + assert [s.name for s in report.signals if s.name == "search_unavailable"] == ["search_unavailable"] diff --git a/.github/actions/contributor-check/vendor/agt/README.md b/.github/actions/contributor-check/vendor/agt/README.md index a044d14..8b53841 100644 --- a/.github/actions/contributor-check/vendor/agt/README.md +++ b/.github/actions/contributor-check/vendor/agt/README.md @@ -57,7 +57,17 @@ The retry is limited to transient failures and reports anything else as itself: | 403 rate limit | Honours `Retry-After`, clamped to 5-60s, within the attempt bound. | | 5xx | Full-jitter exponential backoff within the attempt bound. | | `URLError` | Full-jitter exponential backoff within the attempt bound. | -| Any other HTTP status | Raised on the first attempt. | +| Any other HTTP status | Raised on the first attempt, after a stderr line naming the status and the request path. | + +Every exhausted-retry line names the request path as well, since the action +surfaces only the last stderr line. + +`_search_issues` raises `SearchUnavailable` when the search API answers 422 +"The listed users cannot be searched", which it does for some accounts with +public profiles under any token but their own. `check_contributor` then reports +`UNKNOWN` with a `search_unavailable` signal after the account-shape and +repo-theme checks, rather than crashing or scoring only the checks that could +run. See agentrust-io/.github#56. When a retryable condition outlasts the bound, the final line written to stderr names the cause before the exception propagates. `contributor_check_action.py` diff --git a/.github/actions/contributor-check/vendor/agt/contributor_check.py b/.github/actions/contributor-check/vendor/agt/contributor_check.py index 9c8ace2..7e13f2b 100644 --- a/.github/actions/contributor-check/vendor/agt/contributor_check.py +++ b/.github/actions/contributor-check/vendor/agt/contributor_check.py @@ -116,7 +116,7 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: continue print( f" Rate limited by the GitHub API and still limited after " - f"{_RETRY_MAX_ATTEMPTS} attempts", + f"{_RETRY_MAX_ATTEMPTS} attempts on {path}", file=sys.stderr, ) raise @@ -132,11 +132,13 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: continue print( f" GitHub API returned {exc.code} on all " - f"{_RETRY_MAX_ATTEMPTS} attempts", + f"{_RETRY_MAX_ATTEMPTS} attempts on {path}", file=sys.stderr, ) raise - # Anything else is the API answering. Report it as itself. + # Anything else is the API answering. Report it as itself, and + # name the call: the action surfaces only the last stderr line. + print(f" GitHub API returned {exc.code} on {path}", file=sys.stderr) raise except URLError as exc: # Network-layer failures: DNS, TCP reset, TLS abort, socket @@ -152,15 +154,35 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: continue print( f" Network error ({exc.reason}) on all " - f"{_RETRY_MAX_ATTEMPTS} attempts", + f"{_RETRY_MAX_ATTEMPTS} attempts on {path}", file=sys.stderr, ) raise +class SearchUnavailable(Exception): + """GitHub search refuses to run an ``author:`` query for this account. + + The search API answers 422 "The listed users cannot be searched" for some + accounts whose profiles are otherwise public, under any token other than + the account's own. No issue history can be read for them, so the signals + built on it are not established. See agentrust-io/.github#56. + """ + + def _search_issues(query: str, per_page: int = 30) -> list[dict]: """Search GitHub issues/PRs.""" - data = _api("/search/issues", {"q": query, "per_page": str(per_page)}) + try: + data = _api("/search/issues", {"q": query, "per_page": str(per_page)}) + except HTTPError as exc: + if exc.code == 422: + try: + body = exc.read().decode("utf-8", "replace") + except Exception: + body = "" + if "cannot be searched" in body: + raise SearchUnavailable(query) from exc + raise return data.get("items", []) if data else [] @@ -1054,7 +1076,10 @@ def check_contributor(username: str, target_repo: str | None = None) -> Reputati # Shared data fetches (avoids redundant API calls across checkers) repos = _api(f"/users/{username}/repos", {"per_page": "100", "sort": "created"}) or [] - issues = _search_issues(f"author:{username} is:issue", per_page=100) + try: + issues = _search_issues(f"author:{username} is:issue", per_page=100) + except SearchUnavailable: + issues = None # Run checks with shared data for signal in check_account_shape(user): @@ -1063,6 +1088,21 @@ def check_contributor(username: str, target_repo: str | None = None) -> Reputati for signal in check_repo_themes(username, repos=repos): report.add(signal) + if issues is None: + # Every remaining check reads issue or PR search. Report the account + # as undetermined rather than scoring the checks that could run as if + # they were all of them. + report.add(Signal( + name="search_unavailable", + severity="LOW", + detail=( + "GitHub search refuses author: queries for this account, so " + "spray, credibility, credential and overlap signals were not established" + ), + )) + report.risk = "UNKNOWN" + return report + for signal in check_spray_pattern(username, issues=issues, user_repos=repos): report.add(signal) From f70a94167b1404b4a1a5394f9213476050e065f5 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Mon, 5 Oct 2026 12:55:23 -0700 Subject: [PATCH 2/2] fix(contributor-check): read a refused search as UNKNOWN in the credential and cluster probes Both probes turned every 422 into None, so an account GitHub search refuses read as having no merge or issue history and scored NONE. The overall label was still UNKNOWN through the profile check, but the credential row reported a clean result it never established. A 422 whose body says the user cannot be searched now raises SearchUnavailable in each probe's _api, and the probe reports UNKNOWN. Any other 422 is read as before. Both files leave the unmodified set, so their checksums move out of vendor-integrity and into the README's provenance note. Raised by @Mayur021 on #57. Signed-off-by: Imran Siddique Co-Authored-By: Claude Opus 5.5 --- .../tests/test_probe_search_unavailable.py | 85 +++++++++++++++++++ .../contributor-check/vendor/agt/README.md | 27 ++++-- .../vendor/agt/cluster_detect.py | 33 ++++++- .../vendor/agt/credential_audit.py | 25 +++++- .github/workflows/contributor-check-tests.yml | 2 - 5 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 .github/actions/contributor-check/tests/test_probe_search_unavailable.py diff --git a/.github/actions/contributor-check/tests/test_probe_search_unavailable.py b/.github/actions/contributor-check/tests/test_probe_search_unavailable.py new file mode 100644 index 0000000..d695e51 --- /dev/null +++ b/.github/actions/contributor-check/tests/test_probe_search_unavailable.py @@ -0,0 +1,85 @@ +"""The credential and cluster probes on an account GitHub search refuses. + +Both probes read ``author:`` searches. Their ``_api`` turned every 422 into +``None``, so a refused search read as an empty history and scored NONE, the one +reading that looks clean. A refusal now reports UNKNOWN; any other 422 keeps +its old meaning. Follow-up to agentrust-io/.github#56, raised on #57. +""" +from __future__ import annotations + +import importlib.util +import io +import json +import sys +from pathlib import Path +from urllib.error import HTTPError + +import pytest + +VENDOR = Path(__file__).resolve().parents[1] / "vendor" / "agt" + +REFUSED = ( + b'{"message":"Validation Failed","errors":[{"message":"The listed users cannot be ' + b'searched either because the users do not exist or you do not have permission to ' + b'view the users.","resource":"Search","field":"q","code":"invalid"}]}' +) +OTHER_422 = b'{"message":"Validation Failed"}' + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(f"agt_{name}", VENDOR / f"{name}.py") + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + + +class _Resp: + def __init__(self, payload): + self._payload = json.dumps(payload).encode() + + def read(self): + return self._payload + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + +def _fake_urlopen(search_body: bytes): + def fake(req, timeout=None): + url = req.full_url + if "/search/" in url: + raise HTTPError(url, 422, "err", {}, io.BytesIO(search_body)) + if url.rstrip("/").endswith("/users/someone"): + return _Resp({"login": "someone", "created_at": "2014-05-17T06:30:28Z", + "public_repos": 0, "followers": 0, "following": 0}) + return _Resp([]) + return fake + + +@pytest.fixture(params=["credential_audit", "cluster_detect"]) +def probe(request, monkeypatch): + mod = _load(request.param) + monkeypatch.setattr(mod, "_get_token", lambda: "t") + return request.param, mod + + +def _risk(name, mod): + if name == "credential_audit": + return mod.audit_credentials("someone", "agentrust-io/.github").risk + return mod.detect_cluster("someone").risk_level() + + +def test_refused_search_is_unknown_not_clean(probe, monkeypatch): + name, mod = probe + monkeypatch.setattr(mod, "urlopen", _fake_urlopen(REFUSED)) + assert _risk(name, mod) == "UNKNOWN" + + +def test_other_422_keeps_its_old_reading(probe, monkeypatch): + name, mod = probe + monkeypatch.setattr(mod, "urlopen", _fake_urlopen(OTHER_422)) + assert _risk(name, mod) == "NONE" diff --git a/.github/actions/contributor-check/vendor/agt/README.md b/.github/actions/contributor-check/vendor/agt/README.md index 8b53841..9d318f0 100644 --- a/.github/actions/contributor-check/vendor/agt/README.md +++ b/.github/actions/contributor-check/vendor/agt/README.md @@ -18,15 +18,17 @@ level. See agentrust-io/.github#27. |---|---|---| | `contributor_check.py` | `scripts/contributor_check.py` | modified, see below | | `contributor_check_allowlist.json` | `scripts/contributor_check_allowlist.json` | `d8a5ddbb0695c21b3eae88e6bc85edcda6cbd9721cad3ca9d10fbeeed4fc0636` | -| `credential_audit.py` | `scripts/credential_audit.py` | `9650675038e7720a6d5cdd8be058fa38b2e004cc1e6f7bc81c48ed100e4680e1` | -| `cluster_detect.py` | `scripts/cluster_detect.py` | `da36c10074aa638bf61c5873fbe6cff975533563553689eba5e90329e3d8d7cf` | +| `credential_audit.py` | `scripts/credential_audit.py` | modified, see below | +| `cluster_detect.py` | `scripts/cluster_detect.py` | modified, see below | | `contributor_check_action.py` | `scripts/contributor_check_action.py` | modified, see below | -The first three unmodified files are byte-identical to upstream at that commit. -The checksums above are the upstream bytes, so `sha256sum` here is a one-line -verification for those three. +The unmodified file is byte-identical to upstream at that commit. Its checksum +above is the upstream bytes, so `sha256sum` here is a one-line verification. +`credential_audit.py` was `9650675038e7720a6d5cdd8be058fa38b2e004cc1e6f7bc81c48ed100e4680e1` +and `cluster_detect.py` was `da36c10074aa638bf61c5873fbe6cff975533563553689eba5e90329e3d8d7cf` +upstream before the change below. -## The two local modifications +## Local modifications ### `contributor_check_action.py` @@ -69,6 +71,13 @@ public profiles under any token but their own. `check_contributor` then reports repo-theme checks, rather than crashing or scoring only the checks that could run. See agentrust-io/.github#56. +### `credential_audit.py` and `cluster_detect.py` + +`credential_audit.py` and `cluster_detect.py` recognise the same refusal in +their own `_api` and report `UNKNOWN` for it. Before this, their catch-all for +404 and 422 read a refused search as an empty history, which scored `NONE`. +Any other 422 is still read as before. + When a retryable condition outlasts the bound, the final line written to stderr names the cause before the exception propagates. `contributor_check_action.py` reports the last stderr line, so a rate limit surfaces as a rate limit and a @@ -95,6 +104,6 @@ here. The commit above is provenance, not a sync point, and no resync is planned. Change these files directly, as you would any other file in this repository. -If a change lands in one of the three that remain byte-identical, move it out of -the checksum table above and out of the `vendor-integrity` job, and describe the -change here, the way the two modified files are described. +If a change lands in the file that remains byte-identical, move it out of the +checksum table above and out of the `vendor-integrity` job, and describe the +change here, the way the modified files are described. diff --git a/.github/actions/contributor-check/vendor/agt/cluster_detect.py b/.github/actions/contributor-check/vendor/agt/cluster_detect.py index b250175..70338ec 100644 --- a/.github/actions/contributor-check/vendor/agt/cluster_detect.py +++ b/.github/actions/contributor-check/vendor/agt/cluster_detect.py @@ -57,6 +57,22 @@ def _get_token() -> str: return token +class SearchUnavailable(Exception): + """GitHub search refuses ``author:`` queries for this account. + + Same refusal as in ``contributor_check.py``. Returning ``None`` here would + read as an empty history and score clean. See agentrust-io/.github#56. + """ + + +def _is_search_refusal(exc: HTTPError) -> bool: + try: + body = exc.read().decode("utf-8", "replace") + except Exception: + return False + return "cannot be searched" in body + + def _api(path: str, params: dict[str, str] | None = None) -> Any: url = f"https://api.github.com{path}" if params: @@ -77,6 +93,8 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: print(f" Rate limited, waiting {wait}s...", file=sys.stderr) import time; time.sleep(wait) continue + if exc.code == 422 and _is_search_refusal(exc): + raise SearchUnavailable(path) from exc if exc.code in (404, 422): return None raise @@ -119,6 +137,7 @@ class ClusterReport: accounts: dict[str, AccountInfo] = field(default_factory=dict) edges: list[Edge] = field(default_factory=list) shared_forks: dict[str, list[str]] = field(default_factory=dict) + search_unavailable: bool = False @property def account_count(self) -> int: @@ -129,6 +148,8 @@ def edge_count(self) -> int: return len(self.edges) def risk_level(self) -> str: + if self.search_unavailable: + return "UNKNOWN" if self.account_count >= 5 and self.edge_count >= 8: return "HIGH" elif self.account_count >= 3 and self.edge_count >= 4: @@ -327,6 +348,16 @@ def detect_cluster(seed: str, depth: int = 1) -> ClusterReport: return report report.accounts[seed] = seed_info + try: + _walk(report, seed, depth) + except SearchUnavailable: + # The co-comment and sync-filing passes read author: search. Without + # them the edge count is not established, so no level is reported. + report.search_unavailable = True + return report + + +def _walk(report: ClusterReport, seed: str, depth: int) -> None: visited = {seed} frontier = {seed} @@ -377,8 +408,6 @@ def detect_cluster(seed: str, depth: int = 1) -> ClusterReport: unique_edges.append(edge) report.edges = unique_edges - return report - # --------------------------------------------------------------------------- # Output formatting diff --git a/.github/actions/contributor-check/vendor/agt/credential_audit.py b/.github/actions/contributor-check/vendor/agt/credential_audit.py index 680b804..1a42509 100644 --- a/.github/actions/contributor-check/vendor/agt/credential_audit.py +++ b/.github/actions/contributor-check/vendor/agt/credential_audit.py @@ -59,6 +59,22 @@ def _get_token() -> str: return token +class SearchUnavailable(Exception): + """GitHub search refuses ``author:`` queries for this account. + + Same refusal as in ``contributor_check.py``. Returning ``None`` here would + read as an empty history and score clean. See agentrust-io/.github#56. + """ + + +def _is_search_refusal(exc: HTTPError) -> bool: + try: + body = exc.read().decode("utf-8", "replace") + except Exception: + return False + return "cannot be searched" in body + + def _api(path: str, params: dict[str, str] | None = None) -> Any: url = f"https://api.github.com{path}" if params: @@ -81,6 +97,8 @@ def _api(path: str, params: dict[str, str] | None = None) -> Any: print(f" Rate limited, waiting {wait}s...", file=sys.stderr) import time; time.sleep(wait) continue + if exc.code == 422 and _is_search_refusal(exc): + raise SearchUnavailable(path) from exc if exc.code in (404, 422): return None raise @@ -285,7 +303,12 @@ def audit_credentials(username: str, target_repo: str) -> CredentialAuditReport: report = CredentialAuditReport(username=username, target_repo=target_repo) # Step 1: Find merges - report.merges = find_merges(username, target_repo) + try: + report.merges = find_merges(username, target_repo) + except SearchUnavailable: + # No merge history can be read, which is not the same as none. + report.risk = "UNKNOWN" + return report if not report.merges: report.risk = "NONE" return report diff --git a/.github/workflows/contributor-check-tests.yml b/.github/workflows/contributor-check-tests.yml index 34019f1..fe3e959 100644 --- a/.github/workflows/contributor-check-tests.yml +++ b/.github/workflows/contributor-check-tests.yml @@ -39,7 +39,5 @@ jobs: run: | cd .github/actions/contributor-check/vendor/agt sha256sum -c --strict - <<'SUMS' - 9650675038e7720a6d5cdd8be058fa38b2e004cc1e6f7bc81c48ed100e4680e1 credential_audit.py - da36c10074aa638bf61c5873fbe6cff975533563553689eba5e90329e3d8d7cf cluster_detect.py d8a5ddbb0695c21b3eae88e6bc85edcda6cbd9721cad3ca9d10fbeeed4fc0636 contributor_check_allowlist.json SUMS