Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions EVALUATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -356,11 +356,11 @@ scorer = client.monitor.judge_scorers.builder(
|---|---|---|---|
| `live` | `bool` | `False` | Create the online profile and start scoring live traffic |
| `sample_rate` | `float` | `0.1` | Fraction of traffic actually scored |
| `scope` | `str` | `"trace"` | `"trace"` scores individual traces at ingest; `"session"` scores whole conversations |
| `scope` | `str` | `"trace"` | `"trace"` scores individual traces at ingest; `"session"` scores whole conversations. Non-default values require `live=True` |
| `alert_threshold` | `float \| None` | `5` | A score below this raises a signal. `None` scores without ever raising one |
| `severity` | `str` | `"medium"` | `"low"`, `"medium"`, `"high"` or `"critical"`, applied to signals it raises |
| `agent_ids` | `list[str]` | `None` | Restrict scoring to specific agents instead of the whole workspace |
| `idle_seconds` | `int` | `120` | For `scope="session"`: how long a session must be quiet before it is judged |
| `agent_ids` | `list[str]` | `None` | Restrict scoring to specific agents instead of the whole workspace. Requires `live=True` |
| `idle_seconds` | `int` | `None` (server stores `120`) | For `scope="session"`: how long a session must be quiet before it is judged. Requires `live=True` and `scope="session"` - the builder raises otherwise |

Calibration, tuning and the live-scoring history hang off the same scorer id - the SDK resolves the online profile for you:

Expand Down Expand Up @@ -848,7 +848,7 @@ report = client.evaluations.run(...).execute(my_agent).finalize().analyze()

# Top-level convenience accessors - return None when the metric was not
# enabled on the dataset, or no case has a value yet.
report.average_rating # float | None - same as report.statistics.average_rating
report.average_rating # float | None - None only when no analysis exists; an analyzed run that scored nothing reads 0.0
report.cosine_similarity # float | None - averaged across cases (0-1)
report.jaccard_similarity # float | None - averaged across cases (0-1)
report.bleu_score # float | None - averaged across cases (0-1)
Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ Also see [SDK Developer Docs](https://developers.agentx.so), [API Reference Docs
pip install --upgrade agentx-python
```

Requires Python 3.9 or newer.
Requires Python 3.9 or newer for the core SDK. Some integration extras have higher floors set
by their upstream packages - `[crewai]`, `[autogen]`, and `[databricks]` need Python 3.10+ (`[all]`
therefore does too); the core tracer and every REST surface stay 3.9-compatible.

#### Run the self-host governance suite locally

Expand Down
8 changes: 6 additions & 2 deletions TRACING.md
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ with tracer.trace("support-agent") as span:
span.output = answer
```

`tracer.record_memory(name, operation=..., query=..., output=..., duration_ms=...)` is the after-the-fact form. `operation` is free text - conventionally `"read"` or `"write"` - carried in the span's metadata, while the kind itself stays one value so dashboards and scorers can select all memory activity at once. With no active span the record is dropped (with a debug log) - the pending queue rides the next trace's retrieval steps, and memory content must never feed the RAG judges' retrieval context - so wrap the call in `tracer.trace()`.
`tracer.record_memory(name, operation=..., query=..., output=..., duration_ms=...)` is the after-the-fact form. `operation` is free text - conventionally `"read"` or `"write"` - carried in the span's metadata, while the kind itself stays one value so dashboards and scorers can select all memory activity at once. With no active span the record is dropped (warned once per process, then logged at debug), not queued: there are two pending queues (tool calls ride the next trace's `tool_calls`, retrievals its `retrieval_steps`) and neither fits - `retrieval_steps` feeds the engine's RAG `{context}` extraction, which recalled state must never reach, and no memory-shaped queue has been built yet - so wrap the call in `tracer.trace()`.

---

Expand Down Expand Up @@ -520,7 +520,11 @@ print(pattern.id)

client.monitor.patterns.get(pattern.id) # -> MonitorPattern
client.monitor.patterns.list() # -> list[MonitorPattern]
client.monitor.patterns.update(pattern.id, enabled=False) # sparse update, wire camelCase keys -> MonitorPattern
client.monitor.patterns.update(pattern.id, enabled=False) # sparse update -> MonitorPattern
# update() takes snake_case kwargs (or wire camelCase); an unknown snake_case key raises.
# Sending regex / semantic_prompt / include_terms rebuilds the pattern's conditions for that
# detector kind; exclude_terms, match_mode, or match_target sent alone raise ValueError
# (the engine would silently ignore them) - pass conditions=[...] or a trigger field instead.
client.monitor.patterns.delete(pattern.id) # historical signals remain as history
```

Expand Down
28 changes: 16 additions & 12 deletions agentx/agentx.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ def __init__(
# The api_key is NOT written back into os.environ (it used to be): every sub-client
# below receives it explicitly, and mutating process-global state from a constructor
# re-pointed unrelated code - the same leak the base_url write below had (deep-dive
# round 3, bug #1). Static flows that still read the env (AgentX.list_workforces,
# bare get_headers()) now require the caller to set AGENTX_API_KEY themselves.
# round 3, bug #1). Flows that still read the env (bare get_headers()) now require
# the caller to set AGENTX_API_KEY themselves.
self.api_key = api_key or os.getenv("AGENTX_API_KEY")

# base_url overrides AGENTX_API_BASE_URL env var (and the SDK default). It is
Expand Down Expand Up @@ -147,7 +147,7 @@ def get_agent(self, id: str) -> Agent:
response = requests.get(url, headers=get_headers(self.api_key))
# Check if response was successful
if response.status_code == 200:
return Agent(**response.json())
return Agent(**response.json())._bind(self.api_key, self.base_url)
else:
raise AgentXError(
f"Failed to retrieve agent: {response.reason}. This endpoint is "
Expand All @@ -165,22 +165,26 @@ def list_agents(self) -> List[Agent]:
response = requests.get(url, headers=get_headers(self.api_key))
# Check if response was successful
if response.status_code == 200:
return [Agent(**agent) for agent in response.json()]
return [Agent(**agent)._bind(self.api_key, self.base_url) for agent in response.json()]
else:
raise AgentXError(
f"Failed to list agents: {response.reason}. This endpoint is "
"hosted-platform only - on self-host use client.monitor.agents.list()."
)

@staticmethod
def list_workforces() -> List["Workforce"]:
"""List all workforces/teams. Static, so it reads AGENTX_API_KEY from the environment
directly - the constructor no longer writes ``api_key`` into os.environ, so set the
env var yourself before calling this."""
url = f"{api_base()}/access/teams"
response = requests.get(url, headers=get_headers())
def list_workforces(self) -> List["Workforce"]:
"""List all workforces/teams, each bound to this client's credentials - including each
workforce's ``manager`` and ``agents``, so their calls authenticate the same way.

This used to be documented as a static call (``AgentX.list_workforces()``); that form
was broken (the old staticmethod body referenced ``self`` and raised NameError on any
non-empty response). Construct a client instead - ``AgentX().list_workforces()`` picks
up AGENTX_API_KEY / AGENTX_API_BASE_URL from the environment, which is what the static
form effectively did."""
url = f"{self.base_url or api_base()}/access/teams"
response = requests.get(url, headers=get_headers(self.api_key))
if response.status_code == 200:
return [Workforce(**workforce) for workforce in response.json()]
return [Workforce(**workforce)._bind(self.api_key, self.base_url) for workforce in response.json()]
else:
raise Exception(
f"Failed to list workforces: {response.status_code} - {response.reason}"
Expand Down
4 changes: 2 additions & 2 deletions agentx/integrations/crewai.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,14 +144,14 @@ def _start_task_timing_capture(self, crew: Any = None):
TaskFailedEvent,
TaskStartedEvent,
)
except ImportError:
except Exception: # noqa: BLE001 - crewai import raises TypeError (PEP 604) on py3.9, not just ImportError
from crewai.utilities.events import crewai_event_bus
from crewai.utilities.events.task_events import (
TaskCompletedEvent,
TaskFailedEvent,
TaskStartedEvent,
)
except ImportError:
except Exception: # noqa: BLE001 - crewai import raises TypeError (PEP 604) on py3.9, not just ImportError
if not _warned_no_event_bus:
_warned_no_event_bus = True
logger.warning(
Expand Down
9 changes: 9 additions & 0 deletions agentx/monitor/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
from agentx.monitor.models import MonitorPattern, MonitorProfile, MonitorSignal, SignalOccurrence
from agentx.monitor.patterns import MonitorPatternBuilder, MonitorPatternClient
from agentx.monitor.profile import MonitorProfileClient
from agentx.monitor.review_queue import ReviewQueueClient, ReviewQueueItem
from agentx.monitor.rules import MonitorRule, MonitorRulesClient
from agentx.monitor.scorers import AgentXScorersError, ScorersClient
from agentx.monitor.scorer_groups import AgentXScorerGroupsError, ScorerGroup, ScorerGroupsClient
from agentx.monitor.sessions import MonitorSessionClient
from agentx.monitor.signals import MonitorSignalClient
Expand All @@ -19,6 +22,7 @@
"AgentXJudgeScorersError",
"AgentXMonitorError",
"AgentXScorerGroupsError",
"AgentXScorersError",
"ImprovementGroupsClient",
"JudgeScorer",
"JudgeScorerBuilder",
Expand All @@ -30,10 +34,15 @@
"MonitorPatternClient",
"MonitorProfile",
"MonitorProfileClient",
"MonitorRule",
"MonitorRulesClient",
"MonitorSessionClient",
"MonitorSignal",
"MonitorSignalClient",
"ReviewQueueClient",
"ReviewQueueItem",
"ScorerGroup",
"ScorersClient",
"ScorerGroupsClient",
"SignalOccurrence",
]
48 changes: 48 additions & 0 deletions agentx/monitor/_transport.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Shared HTTP transport for the monitor sub-clients that own their ``_request`` (scorers,
judge_scorers, scorer_groups, improvement_groups): one retry schedule mirroring
``MonitorClient._request``, so ``retry=False`` means the same thing everywhere the client.py
comment promises it ("retry=False for ANY non-idempotent write")."""

from __future__ import annotations

import logging
import time
from typing import Any, Optional

import requests

logger = logging.getLogger(__name__)

# Same schedule as MonitorClient._request (agentx/monitor/client.py).
_RETRYABLE_STATUS = {429, 500, 502, 503, 504}
_RETRY_BACKOFF = [1.0, 2.0, 4.0]


def request_with_retries(
method: str, url: str, *, retry: bool = True, **kwargs: Any
) -> requests.Response:
"""``requests.request`` with MonitorClient's transport posture: when ``retry`` is true,
connection errors and retryable statuses (429/5xx) walk the backoff schedule; the last
response (whatever its status) is returned for the caller's own error taxonomy.

``retry=False`` is single-shot - for non-idempotent writes (creates, deletes) and
judge-billing POSTs, where a client-side timeout must not fire the same work twice.
Transport errors keep their ``requests`` exception type (callers guard on
``requests.Timeout`` for judge-billing endpoints)."""
schedule = [0.0] + _RETRY_BACKOFF if retry else [0.0]
last_exc: Optional[Exception] = None
for attempt, wait in enumerate(schedule):
if wait:
time.sleep(wait)
try:
resp = requests.request(method, url, **kwargs)
except requests.RequestException as e:
last_exc = e
logger.debug("Request error (attempt %d): %s", attempt + 1, e)
continue
if retry and resp.status_code in _RETRYABLE_STATUS and attempt < len(schedule) - 1:
logger.debug("Retryable status %d (attempt %d)", resp.status_code, attempt + 1)
continue
return resp
assert last_exc is not None # every non-raising path returned above
raise last_exc
8 changes: 5 additions & 3 deletions agentx/monitor/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,11 @@ def _api_root(self) -> str:
def _request(
self, method: str, path: str, timeout: int = 30, base: Optional[str] = None, retry: bool = True, **kwargs
) -> Any:
# retry=False for non-idempotent judge-spending POSTs (sweep, coherence, portability,
# tuning): a client-side timeout must not fire the same LLM-billing work a second time
# while the first invocation is still running server-side. Same precedent as
# retry=False for ANY non-idempotent write - duplicating creates, deletes (a lost
# response + retry turns success into a spurious 404), and judge-spending POSTs
# (sweep, coherence, portability, tuning: a client-side timeout must not fire the
# same LLM-billing work twice while the first invocation still runs server-side).
# The judge list is the example set, not the rule. Same precedent as
# EvaluationsClient._request / analyze_run.
url = f"{base or self._base_url}{path}"
last_exc: Optional[Exception] = None
Expand Down
20 changes: 16 additions & 4 deletions agentx/monitor/improvement_groups.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import requests

from agentx.monitor._transport import request_with_retries
from agentx.util import api_base, get_headers
from agentx.exceptions import AgentXError, AgentXAuthError, AgentXValidationError

Expand Down Expand Up @@ -45,7 +46,9 @@ def __init__(
self._workspace_id = workspace_id
self._base_url = (base_url or api_base()).rstrip("/")

def _request(self, method: str, path: str, json: Any = None, timeout: int = 120) -> Any:
def _request(
self, method: str, path: str, json: Any = None, timeout: int = 120, retry: bool = True
) -> Any:
params = None
if self._workspace_id:
# Mirrors MonitorClient._workspace_params/_with_workspace: GETs (and DELETEs)
Expand All @@ -57,9 +60,12 @@ def _request(self, method: str, path: str, json: Any = None, timeout: int = 120)
json = {**json, "workspaceId": self._workspace_id}
else:
params = {"workspaceId": self._workspace_id}
resp = requests.request(
# retry=False for ANY non-idempotent write (member deletes, the report-generating
# POST) - MonitorClient._request's posture, via the shared monitor transport.
resp = request_with_retries(
method,
f"{self._base_url}/agent-monitoring{path}",
retry=retry,
headers={**get_headers(self._api_key), "Content-Type": "application/json"},
json=json,
params=params,
Expand Down Expand Up @@ -92,7 +98,9 @@ def get(self, group_id: str) -> Dict[str, Any]:

def remove_member(self, group_id: str, member_id: str) -> None:
"""Prune a member before spending the group (a confirm that turned out uninteresting)."""
self._request("DELETE", f"/improvement-groups/{group_id}/members/{member_id}")
# retry=False: a lost response + transport retry would turn a successful delete
# into a spurious 404.
self._request("DELETE", f"/improvement-groups/{group_id}/members/{member_id}", retry=False)

def generate_report(self, group_id: str, model: Optional[str] = None) -> Dict[str, Any]:
"""Spend the group: one real LLM call clustering the confirmed failures into issues
Expand All @@ -101,7 +109,11 @@ def generate_report(self, group_id: str, model: Optional[str] = None) -> Dict[st
payload: Dict[str, Any] = {}
if model is not None:
payload["model"] = model
return self._request("POST", f"/improvement-groups/{group_id}/report", json=payload, timeout=300)["report"]
# retry=False: spends the group (real LLM billing) - a client-side timeout must not
# fire the same generation twice while the first still runs server-side.
return self._request(
"POST", f"/improvement-groups/{group_id}/report", json=payload, timeout=300, retry=False
)["report"]

def list_reports(self) -> List[Dict[str, Any]]:
return self._request("GET", "/improvement-reports").get("improvementReports", [])
Expand Down
Loading
Loading