From 0bd258b1dc07dd5613b48a2aeb62f5c3708b543a Mon Sep 17 00:00:00 2001 From: olaams-tech Date: Sat, 17 Oct 2026 17:24:00 +0100 Subject: [PATCH 1/2] feat(usage): add ledger instrumentation, source breakdown, and currency display Add the central UsageRecord ledger and reversible migration, instrument project chat (streamed and non-streamed) and project analysis with project/workspace scope and an estimated flag, expose a per-user breakdown by source API, and format every cost figure in a configurable currency with an explicit estimate label. Refs #429, #432, #451, #441. Adds the ledger foundation required by the Phase 10 instrumentation issues. --- .env.example | 7 + app/chat/api.py | 27 +- app/config.py | 9 +- app/models/__init__.py | 2 + app/models/usage_record.py | 95 ++++++ app/services/currency.py | 128 ++++++++ app/services/project_analysis.py | 129 ++++++-- app/services/usage_ledger.py | 298 ++++++++++++++++++ app/static/js/currency.js | 85 +++++ app/static/js/project.js | 5 +- app/templates/base.html | 1 + app/workspaces/routes.py | 51 ++- .../versions/c4d5e6f7a8b9_add_usage_ledger.py | 60 ++++ tests/js/currency.test.js | 54 ++++ tests/test_currency.py | 112 +++++++ tests/test_migrations.py | 43 ++- tests/test_usage_ledger.py | 291 +++++++++++++++++ 17 files changed, 1368 insertions(+), 29 deletions(-) create mode 100644 app/models/usage_record.py create mode 100644 app/services/currency.py create mode 100644 app/services/usage_ledger.py create mode 100644 app/static/js/currency.js create mode 100644 migrations/versions/c4d5e6f7a8b9_add_usage_ledger.py create mode 100644 tests/js/currency.test.js create mode 100644 tests/test_currency.py create mode 100644 tests/test_usage_ledger.py diff --git a/.env.example b/.env.example index c612985..e9a6dca 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,13 @@ LLM_CACHE_ENABLED=1 LLM_CACHE_TTL=300 LLM_CACHE_MAX_ENTRIES=256 +# Display currency for cost estimates (Phase 10, issue #441). Costs are always +# estimates, so every figure shows the currency code explicitly. Keep the +# configured estimate/price rates in this currency when overriding the default. +CURRENCY_CODE=USD +CURRENCY_SYMBOL=$ +CURRENCY_DECIMAL_PLACES=4 + # Plugin manifest trust (optional). Unsigned plugins remain allowed by default. # Set PLUGIN_TRUST_POLICY=required to reject unsigned or invalid signatures. # JSON object mapping publisher ids to base64 Ed25519 public keys. diff --git a/app/chat/api.py b/app/chat/api.py index 4740270..c2c4d85 100644 --- a/app/chat/api.py +++ b/app/chat/api.py @@ -8,6 +8,7 @@ POST /api/conversations//messages send a message (LLM reply) DELETE /api/conversations/ delete (cascade) GET /api/usage per-user token usage summary + GET /api/usage/breakdown usage and cost by source All routes require authentication and are owner-scoped. Errors use RFC 7807 (``application/problem+json``) documents so API clients never receive HTML error @@ -26,7 +27,8 @@ from app.chat import routes as chat_routes from app.extensions import db from app.models import Conversation, Message -from app.services import ratelimit, streaming +from app.services import ratelimit, streaming, usage_ledger +from app.services.currency import get_currency_config from app.services.llm import LLMProviderError, provider_status from app.services.provider_config import ProviderSettingsError, apply_settings, build_provider from app.services.providers.retry import RetryingProvider @@ -294,3 +296,26 @@ def usage_summary(): "total_tokens": total, } ) + + +@bp.route("/usage/breakdown", methods=["GET"]) +@_login_required +@_rate_limit("usage") +def usage_breakdown(): + """Return usage counts and estimated cost grouped by ledger source. + + The authenticated user's records are always the outer scope, so an optional + ``workspace_id`` or ``project_id`` filter can never expose another account's + usage. Unknown historical sources are grouped under ``unknown`` rather than + dropped or causing a 500. + """ + workspace_id = request.args.get("workspace_id", type=int) + project_id = request.args.get("project_id", type=int) + payload = usage_ledger.breakdown_payload( + user_id=current_user.id, + workspace_id=workspace_id, + project_id=project_id, + ) + payload["currency"] = get_currency_config() + payload["cost_is_estimate"] = True + return jsonify(payload) diff --git a/app/config.py b/app/config.py index ad867c8..6137652 100644 --- a/app/config.py +++ b/app/config.py @@ -98,8 +98,15 @@ class Config: LLM_CACHE_TTL = int(os.getenv("LLM_CACHE_TTL", "300")) LLM_CACHE_MAX_ENTRIES = int(os.getenv("LLM_CACHE_MAX_ENTRIES", "256")) # Pre-flight token/cost estimate (issue #104). The rate is only used to turn - # an estimated token count into a rough USD figure shown before sending. + # an estimated token count into a rough figure shown before sending. LLM_ESTIMATE_USD_PER_1K_TOKENS = float(os.getenv("LLM_ESTIMATE_USD_PER_1K_TOKENS", "0.002")) + # Display currency for every cost figure (issue #441). Defaults to USD; the + # code is always shown next to the symbol so a figure is never ambiguous. + # Operators that change the code are responsible for keeping the configured + # price/estimate rates in the same currency. + CURRENCY_CODE = os.getenv("CURRENCY_CODE", "USD") + CURRENCY_SYMBOL = os.getenv("CURRENCY_SYMBOL", "$") + CURRENCY_DECIMAL_PLACES = int(os.getenv("CURRENCY_DECIMAL_PLACES", "4")) # Structured chat audit logging (issue #17). ``CHAT_AUDIT_LOG_LEVEL`` may be # any logging level or OFF to disable. CHAT_AUDIT_LOG_ENABLED = os.getenv("CHAT_AUDIT_LOG_ENABLED", "1") == "1" diff --git a/app/models/__init__.py b/app/models/__init__.py index 6dbf994..9e384b4 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -31,6 +31,7 @@ from app.models.review_config import ReviewConfig from app.models.review_finding import ReviewFinding from app.models.stellar_security_finding import StellarSecurityFinding +from app.models.usage_record import UsageRecord from app.models.user import User from app.models.workspace import Workspace from app.models.workspace_member import WorkspaceMember @@ -65,6 +66,7 @@ "ReviewConfig", "ReviewFinding", "StellarSecurityFinding", + "UsageRecord", "User", "Workspace", "WorkspaceInvitation", diff --git a/app/models/usage_record.py b/app/models/usage_record.py new file mode 100644 index 0000000..37df57e --- /dev/null +++ b/app/models/usage_record.py @@ -0,0 +1,95 @@ +"""Central usage ledger record (Phase 10). + +Each row is an immutable accounting fact for one LLM operation: how many +tokens were consumed, which source produced the work, and which user, +workspace, project and conversation it belongs to. Prompts and completions are +deliberately **not** stored here; only metadata and counts are. + +Usage is scoped with nullable foreign keys so a record never has to be deleted +when a container is removed: + +- deleting a user cascades to their ledger rows; +- deleting a workspace, project or conversation leaves the accounting row in + place and nulls the corresponding reference (``ON DELETE SET NULL``). +""" + +from datetime import UTC, datetime + +from app.extensions import db + + +class UsageRecord(db.Model): + """One append-only usage entry in the central ledger.""" + + __tablename__ = "usage_records" + + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column( + db.Integer, + db.ForeignKey("users.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + workspace_id = db.Column( + db.Integer, + db.ForeignKey("workspaces.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + project_id = db.Column( + db.Integer, + db.ForeignKey("projects.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + conversation_id = db.Column( + db.Integer, + db.ForeignKey("conversations.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + #: Operation family: chat, analysis, review, tools or project_chat. + #: Unknown values are preserved so readers can group them safely. + source = db.Column(db.String(40), nullable=False, default="unknown", index=True) + provider = db.Column(db.String(50), nullable=True) + model = db.Column(db.String(100), nullable=True) + prompt_tokens = db.Column(db.Integer, nullable=False, default=0) + completion_tokens = db.Column(db.Integer, nullable=False, default=0) + total_tokens = db.Column(db.Integer, nullable=False, default=0) + #: True when any token count was estimated rather than reported by the + #: provider (streamed replies, the offline mock provider, pre-flight work). + tokens_estimated = db.Column(db.Boolean, nullable=False, default=True) + #: Estimated cost in the configured display currency. ``None`` means the + #: provider/model has no known price; it is never silently reported as zero. + cost_estimate = db.Column(db.Float, nullable=True) + created_at = db.Column( + db.DateTime(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + index=True, + ) + + def to_dict(self) -> dict: + """Serialize the record for JSON responses, logs and exports.""" + return { + "id": self.id, + "user_id": self.user_id, + "workspace_id": self.workspace_id, + "project_id": self.project_id, + "conversation_id": self.conversation_id, + "source": self.source, + "provider": self.provider, + "model": self.model, + "prompt_tokens": self.prompt_tokens, + "completion_tokens": self.completion_tokens, + "total_tokens": self.total_tokens, + "tokens_estimated": bool(self.tokens_estimated), + "cost_estimate": self.cost_estimate, + "created_at": self.created_at.isoformat() if self.created_at else None, + } + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return ( + f"" + ) diff --git a/app/services/currency.py b/app/services/currency.py new file mode 100644 index 0000000..f0efa5c --- /dev/null +++ b/app/services/currency.py @@ -0,0 +1,128 @@ +"""Currency configuration and cost formatting (Phase 10, issue #441). + +The ledger and price table work in one configured display currency, defaulting +to USD. Cost figures are estimates (see :mod:`app.services.pricing`), so every +formatted value includes both the currency code and an ``estimate`` marker made +available to callers. + +Formatting is centralised here instead of being duplicated in templates and +JavaScript so every cost figure uses the same symbol, precision and code. +""" + +from __future__ import annotations + +from decimal import ROUND_HALF_UP, Decimal, InvalidOperation + +DEFAULT_CURRENCY_CODE = "USD" +DEFAULT_CURRENCY_SYMBOL = "$" +DEFAULT_DECIMAL_PLACES = 4 + +#: Suffix that makes it unambiguous that a figure is not an invoice amount. +ESTIMATE_SUFFIX = "estimate" + + +def _clean_code(value) -> str: + code = str(value or "").strip().upper() + return code[:8] or DEFAULT_CURRENCY_CODE + + +def _clean_symbol(value) -> str: + symbol = str(value or "").strip() + return symbol[:8] + + +def _clean_places(value) -> int: + try: + places = int(value) + except (TypeError, ValueError): + return DEFAULT_DECIMAL_PLACES + return min(max(places, 0), 8) + + +def normalize_currency(config: dict | None = None) -> dict: + """Return a complete, validated currency configuration. + + Missing or malformed values fall back to USD rather than failing a page + render or API request. Unknown keys are ignored. + """ + config = config or {} + code = _clean_code(config.get("code")) + symbol = _clean_symbol(config.get("symbol", DEFAULT_CURRENCY_SYMBOL)) + places = _clean_places(config.get("decimal_places", DEFAULT_DECIMAL_PLACES)) + return {"code": code, "symbol": symbol, "decimal_places": places} + + +def get_currency_config(app=None) -> dict: + """Read the configured display currency from the Flask application. + + Falls back to the USD defaults when called outside an application context + (for example from a unit test or a formatting helper), so formatting never + fails merely because no app is bound. + """ + if app is None: + from flask import current_app + + try: + config = current_app.config + except RuntimeError: + config = {} + else: + config = app.config + return normalize_currency( + { + "code": config.get("CURRENCY_CODE", DEFAULT_CURRENCY_CODE), + "symbol": config.get("CURRENCY_SYMBOL", DEFAULT_CURRENCY_SYMBOL), + "decimal_places": config.get("CURRENCY_DECIMAL_PLACES", DEFAULT_DECIMAL_PLACES), + } + ) + + +def format_cost(amount, config: dict | None = None) -> str: + """Format ``amount`` with the configured currency symbol, code and precision. + + ``amount`` must be a finite, non-negative number. Values are rounded + half-up to the configured number of decimal places, then the code is shown + explicitly (for example ``$0.0025 USD``) so the display is never ambiguous. + """ + currency = normalize_currency(config) if config is not None else get_currency_config() + try: + value = Decimal(str(amount if amount is not None else 0)) + except (InvalidOperation, ValueError, TypeError) as exc: + raise ValueError("Cost amount must be numeric.") from exc + if not value.is_finite(): + raise ValueError("Cost amount must be finite.") + if value < 0: + raise ValueError("Cost amount must be non-negative.") + + quantizer = Decimal(1).scaleb(-currency["decimal_places"]) + rounded = value.quantize(quantizer, rounding=ROUND_HALF_UP) + number = f"{rounded:,.{currency['decimal_places']}f}" + symbol = currency["symbol"] + if symbol: + return f"{symbol}{number} {currency['code']}" + return f"{number} {currency['code']}" + + +def format_estimate(tokens, amount, config: dict | None = None) -> str: + """Format a pre-flight estimate with tokens and an explicit cost label.""" + currency = normalize_currency(config) if config is not None else get_currency_config() + try: + token_count = max(0, int(tokens or 0)) + except (TypeError, ValueError): + token_count = 0 + return ( + f"Estimated: ~{token_count:,} tokens " + f"({format_cost(amount, currency)}; {ESTIMATE_SUFFIX})" + ) + + +__all__ = [ + "DEFAULT_CURRENCY_CODE", + "DEFAULT_CURRENCY_SYMBOL", + "DEFAULT_DECIMAL_PLACES", + "ESTIMATE_SUFFIX", + "format_cost", + "format_estimate", + "get_currency_config", + "normalize_currency", +] diff --git a/app/services/project_analysis.py b/app/services/project_analysis.py index 2db8304..f3d1ce5 100644 --- a/app/services/project_analysis.py +++ b/app/services/project_analysis.py @@ -10,6 +10,7 @@ from __future__ import annotations +import contextvars import json import re @@ -21,8 +22,10 @@ ROLE_VIEWER, STATUS_ACTIVE, ) +from app.services import token_usage +from app.services.currency import format_cost, get_currency_config from app.services.llm import LLMProviderError, get_provider -from app.services.llm_cache import cached_complete +from app.services.llm_cache import cached_chat from app.services.permissions import assert_content_access from app.services.stellar_detection import ( SOROBAN_CRATES, @@ -243,14 +246,70 @@ def _clip(text: str, limit: int) -> str: return text[:limit] + "\n…[context truncated]" +#: Last completion performed by ``_run`` in this request/context. It lets +#: ``analyze_project`` attach real provider-reported counts (or the estimates +#: filled in by ``token_usage``) to the result without changing every analysis +#: helper's return type. +_LAST_COMPLETION: contextvars.ContextVar = contextvars.ContextVar( + "project_analysis_last_completion", default=None +) + + +def _provider_completion(messages: list[dict], *, user=None, no_cache: bool = False): + """Call the configured provider and return ``(response, provider)``.""" + provider = get_provider(user=user) + if user is not None: + response = cached_chat(user, messages, provider=provider, no_cache=no_cache) + else: + response = provider.chat(messages) + return response, provider + + +def _usage_from_response(response, provider, messages) -> dict: + """Return ledger-ready token usage plus provider/model metadata.""" + usage = token_usage.usage_from_response(response, messages) + name = getattr(provider, "name", None) + models = getattr(provider, "models", ()) or () + model = getattr(response, "model", None) or getattr(provider, "model", None) + if not model and models: + model = models[0] + # The offline provider derives counts from text and never reports vendor + # usage; treat it as estimated. Real providers may report only one side. + estimated = ( + name == "mock" + or getattr(response, "prompt_tokens", None) is None + or getattr(response, "completion_tokens", None) is None + ) + return { + **usage, + "provider": name, + "model": model, + "tokens_estimated": bool(estimated), + } + + +def _with_usage(result: dict) -> dict: + """Attach usage metadata captured by the most recent ``_run`` call.""" + completion = _LAST_COMPLETION.get() + if completion is not None: + response, provider, messages = completion + result["usage"] = _usage_from_response(response, provider, messages) + return result + + def _run(prompt: str, *, system: str = _PROJECT_SYSTEM) -> str: """Run a single completion with the configured provider.""" - return _complete( - [ - {"role": "system", "content": system}, - {"role": "user", "content": prompt}, - ] - ) + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": prompt}, + ] + try: + response, provider = _provider_completion(messages) + except LLMProviderError as exc: + _LAST_COMPLETION.set(None) + return f"[analysis unavailable: {exc}]" + _LAST_COMPLETION.set((response, provider, messages)) + return response.content def _complete(messages: list[dict], *, user=None, no_cache: bool = False) -> str: @@ -261,10 +320,8 @@ def _complete(messages: list[dict], *, user=None, no_cache: bool = False) -> str directly. """ try: - if user is not None: - return cached_complete(user, messages, no_cache=no_cache) - provider = get_provider() - return provider.complete(messages) + response, _ = _provider_completion(messages, user=user, no_cache=no_cache) + return response.content except LLMProviderError as exc: return f"[analysis unavailable: {exc}]" @@ -707,9 +764,19 @@ def chat_with_project( pinned = _pinned_paths(project, attachments, history) context = build_context(project, question, attachments=pinned) messages = build_messages(project, question, history, attachments=pinned) + try: + response, provider = _provider_completion(messages, user=user, no_cache=no_cache) + except LLMProviderError as exc: + analysis = f"[analysis unavailable: {exc}]" + usage = token_usage.usage_from_text(token_usage.messages_text(messages), analysis) + usage.update({"provider": None, "model": None, "tokens_estimated": True}) + else: + analysis = response.content + usage = _usage_from_response(response, provider, messages) return { "context_paths": context["paths"], - "analysis": _complete(messages, user=user, no_cache=no_cache), + "analysis": analysis, + "usage": usage, } @@ -730,13 +797,30 @@ def estimate_tokens(text: str | None) -> int: def _estimate_cost(tokens: int) -> float: - """Estimated USD cost for ``tokens`` using the configured per-1k rate.""" + """Estimated cost for ``tokens`` using the configured per-1k rate.""" from flask import current_app rate = float(current_app.config.get("LLM_ESTIMATE_USD_PER_1K_TOKENS", 0.0) or 0.0) return round(tokens / 1000 * rate, 6) +def _cost_payload(tokens: int) -> dict: + """Return the estimated cost in both raw and consistently formatted form. + + The configured currency is included in the payload so clients never have to + guess which currency a figure is in (issue #441). + """ + amount = _estimate_cost(tokens) + currency = get_currency_config() + return { + "estimated_cost_usd": amount, + "estimated_cost": amount, + "currency": currency, + "estimated_cost_display": format_cost(amount, currency), + "cost_is_estimate": True, + } + + def estimate_chat( project, question: str, @@ -755,7 +839,7 @@ def estimate_chat( return { "mode": "chat", "prompt_tokens": tokens, - "estimated_cost_usd": _estimate_cost(tokens), + **_cost_payload(tokens), } @@ -806,7 +890,7 @@ def estimate_analysis(project, kind: str) -> dict: "mode": "analysis", "kind": kind, "prompt_tokens": tokens, - "estimated_cost_usd": _estimate_cost(tokens), + **_cost_payload(tokens), } @@ -816,6 +900,7 @@ def analyze_project(project, kind: str) -> dict: kind = (kind or "").strip().lower() if kind not in ANALYSIS_KINDS: kind = "architecture" + _LAST_COMPLETION.set(None) structure = project_structure(project) budget = _budget() @@ -899,17 +984,17 @@ def analyze_project(project, kind: str) -> dict: Base everything on the files shown and mark [CONFIRMED] vs [SUGGESTION]. """ elif kind == "quality": - return analyze_code_quality(project) + return _with_usage(analyze_code_quality(project)) elif kind == "security": - return analyze_security(project) + return _with_usage(analyze_security(project)) elif kind == "code_review": - return analyze_code_review(project) + return _with_usage(analyze_code_review(project)) elif kind == "stellar": - return analyze_stellar_project(project) + return _with_usage(analyze_stellar_project(project)) elif kind == "stellar_security": - return analyze_stellar_security(project) + return _with_usage(analyze_stellar_security(project)) elif kind == "stellar_config": - return analyze_stellar_config(project) + return _with_usage(analyze_stellar_config(project)) else: # dependencies inventory = dependency_inventory(project) if inventory: @@ -940,7 +1025,7 @@ def analyze_project(project, kind: str) -> dict: if signals.is_stellar: prompt += _soroban_dependency_context(inventory) - return {"kind": kind, "analysis": _run(prompt)} + return _with_usage({"kind": kind, "analysis": _run(prompt)}) def analyze_code_quality(project) -> dict: diff --git a/app/services/usage_ledger.py b/app/services/usage_ledger.py new file mode 100644 index 0000000..f7c1d0e --- /dev/null +++ b/app/services/usage_ledger.py @@ -0,0 +1,298 @@ +"""Central usage ledger service (Phase 10). + +The ledger is append-only: callers add :class:`~app.models.UsageRecord` rows and +readers aggregate them. This module keeps the normalisation and aggregation +rules in one place so project chat, project analysis, dashboards and future +features (reviews, tools, budgets) all report the same numbers. + +Only metadata is stored. Prompts and completions are never written here. +""" + +from __future__ import annotations + +from typing import Any + +from app.extensions import db +from app.models import UsageRecord +from app.services import pricing + +#: Operation families understood by the ledger. Unknown values are grouped +#: under :data:`UNKNOWN_SOURCE` instead of crashing a dashboard. +SOURCE_CHAT = "chat" +SOURCE_ANALYSIS = "analysis" +SOURCE_REVIEW = "review" +SOURCE_TOOLS = "tools" +SOURCE_PROJECT_CHAT = "project_chat" +SOURCES = ( + SOURCE_CHAT, + SOURCE_ANALYSIS, + SOURCE_REVIEW, + SOURCE_TOOLS, + SOURCE_PROJECT_CHAT, +) +UNKNOWN_SOURCE = "unknown" + + +def normalize_source(source: Any) -> str | None: + """Return the lowercase source label, or ``None`` when it is missing.""" + value = str(source or "").strip().lower() + return value[:40] or None + + +def source_bucket(source: Any) -> str: + """Group a raw source into a known bucket, falling back to ``unknown``.""" + value = normalize_source(source) + if value in SOURCES: + return value + return UNKNOWN_SOURCE + + +def _non_negative_int(value, name: str) -> int: + """Coerce a count to a non-negative integer, rejecting invalid input.""" + if value in (None, ""): + return 0 + try: + count = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be an integer.") from exc + if count < 0: + raise ValueError(f"{name} must be non-negative.") + return count + + +def normalize_usage(prompt_tokens=0, completion_tokens=0, total_tokens=None) -> dict[str, int]: + """Return validated prompt/completion/total token counts. + + ``total_tokens`` is derived from the two parts when it is omitted. Explicit + values are validated as well, so a malformed provider payload can never + write a negative count into the ledger. + """ + prompt = _non_negative_int(prompt_tokens, "prompt_tokens") + completion = _non_negative_int(completion_tokens, "completion_tokens") + total = ( + prompt + completion + if total_tokens is None + else _non_negative_int(total_tokens, "total_tokens") + ) + return { + "prompt_tokens": prompt, + "completion_tokens": completion, + "total_tokens": total, + } + + +def _object_id(value, attribute: str = "id"): + if value is None: + return None + return getattr(value, attribute, None) + + +def current_provider_metadata(user=None) -> tuple[str | None, str | None]: + """Return ``(provider_name, model)`` for the active provider, if resolvable. + + Instrumentation must never break the operation it is recording, so any + configuration error simply yields ``(None, None)`` and cost stays unknown. + """ + try: + from app.services.providers import get_provider + + provider = get_provider(user=user) + name = getattr(provider, "name", None) + model = getattr(provider, "model", None) + if not model: + models = getattr(provider, "models", ()) or () + model = models[0] if models else None + return name, model + except Exception: # pragma: no cover - defensive: never break instrumentation + return None, None + + +def estimate_cost(provider, model, prompt_tokens: int, completion_tokens: int) -> float | None: + """Estimate cost with the default price table, returning ``None`` if unknown.""" + if not provider or not model: + return None + return pricing.estimate_cost_usd( + provider, + model, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + + +def record_usage( + *, + source: str, + prompt_tokens=0, + completion_tokens=0, + total_tokens=None, + user=None, + user_id=None, + workspace=None, + workspace_id=None, + project=None, + project_id=None, + conversation=None, + conversation_id=None, + provider: str | None = None, + model: str | None = None, + tokens_estimated: bool = True, + cost_estimate=None, + commit: bool = True, +) -> UsageRecord: + """Append one usage record and return it. + + Scope objects take precedence over their explicit id counterparts. Cost is + estimated from the configured price table when the caller does not supply a + value; an unknown provider/model leaves it as ``None`` rather than zero. + """ + counts = normalize_usage(prompt_tokens, completion_tokens, total_tokens) + provider = (str(provider).strip()[:50] or None) if provider else None + model = (str(model).strip()[:100] or None) if model else None + + if cost_estimate is None: + cost_estimate = estimate_cost( + provider, model, counts["prompt_tokens"], counts["completion_tokens"] + ) + else: + try: + cost_estimate = float(cost_estimate) + except (TypeError, ValueError) as exc: + raise ValueError("cost_estimate must be numeric.") from exc + if cost_estimate < 0: + raise ValueError("cost_estimate must be non-negative.") + + record = UsageRecord( + user_id=_object_id(user) or user_id, + workspace_id=_object_id(workspace) or workspace_id or _object_id(project, "workspace_id"), + project_id=_object_id(project) or project_id, + conversation_id=_object_id(conversation) or conversation_id, + source=normalize_source(source) or UNKNOWN_SOURCE, + provider=provider, + model=model, + prompt_tokens=counts["prompt_tokens"], + completion_tokens=counts["completion_tokens"], + total_tokens=counts["total_tokens"], + tokens_estimated=bool(tokens_estimated), + cost_estimate=cost_estimate, + ) + db.session.add(record) + if commit: + db.session.commit() + return record + + +def summarize_records(records) -> dict: + """Aggregate counts and costs for an iterable of records.""" + summary = { + "count": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "cost_estimate": 0.0, + "priced_records": 0, + "estimated_records": 0, + } + for record in records: + summary["count"] += 1 + summary["prompt_tokens"] += int(getattr(record, "prompt_tokens", 0) or 0) + summary["completion_tokens"] += int(getattr(record, "completion_tokens", 0) or 0) + summary["total_tokens"] += int(getattr(record, "total_tokens", 0) or 0) + if getattr(record, "tokens_estimated", False): + summary["estimated_records"] += 1 + cost = getattr(record, "cost_estimate", None) + if cost is not None: + summary["cost_estimate"] += float(cost) + summary["priced_records"] += 1 + summary["cost_estimate"] = round(summary["cost_estimate"], 6) + return summary + + +def query_records( + *, + user_id=None, + workspace_id=None, + project_id=None, + source=None, +) -> list[UsageRecord]: + """Fetch ledger rows with optional owner/scope filters.""" + query = UsageRecord.query + if user_id is not None: + query = query.filter(UsageRecord.user_id == user_id) + if workspace_id is not None: + query = query.filter(UsageRecord.workspace_id == workspace_id) + if project_id is not None: + query = query.filter(UsageRecord.project_id == project_id) + if source is not None: + query = query.filter(UsageRecord.source == normalize_source(source)) + return query.order_by(UsageRecord.created_at, UsageRecord.id).all() + + +def breakdown_by_source( + records=None, + *, + user_id=None, + workspace_id=None, + project_id=None, +) -> dict[str, dict]: + """Aggregate usage and cost by source. + + All known sources are returned (zeroed when unused) so consumers get a stable + shape. Rows with a missing or unrecognised ``source`` are grouped under + ``unknown`` only when such rows exist, which keeps malformed historical data + visible without breaking the response. + """ + if records is None: + records = query_records(user_id=user_id, workspace_id=workspace_id, project_id=project_id) + + buckets: dict[str, list] = {source: [] for source in SOURCES} + unknown = [] + for record in records: + bucket = source_bucket(getattr(record, "source", None)) + if bucket == UNKNOWN_SOURCE: + unknown.append(record) + else: + buckets.setdefault(bucket, []).append(record) + + result = {source: summarize_records(buckets[source]) for source in SOURCES} + if unknown: + result[UNKNOWN_SOURCE] = summarize_records(unknown) + return result + + +def breakdown_payload( + *, + user_id=None, + workspace_id=None, + project_id=None, +) -> dict: + """Build the JSON-friendly payload used by the breakdown API.""" + records = query_records(user_id=user_id, workspace_id=workspace_id, project_id=project_id) + by_source = breakdown_by_source(records) + totals = summarize_records(records) + return { + "by_source": by_source, + "sources": [{"source": source, **stats} for source, stats in by_source.items()], + "totals": totals, + "known_sources": list(SOURCES), + } + + +__all__ = [ + "SOURCES", + "SOURCE_ANALYSIS", + "SOURCE_CHAT", + "SOURCE_PROJECT_CHAT", + "SOURCE_REVIEW", + "SOURCE_TOOLS", + "UNKNOWN_SOURCE", + "breakdown_by_source", + "breakdown_payload", + "current_provider_metadata", + "estimate_cost", + "normalize_source", + "normalize_usage", + "query_records", + "record_usage", + "source_bucket", + "summarize_records", +] diff --git a/app/static/js/currency.js b/app/static/js/currency.js new file mode 100644 index 0000000..da6dcb6 --- /dev/null +++ b/app/static/js/currency.js @@ -0,0 +1,85 @@ +// AI Code Assistant — consistent currency and cost formatting (issue #441). +// +// Cost figures are estimates, never invoice amounts. This module is +// dependency-free and DOM-free so the browser and the Vitest suite use the +// exact same formatting rules. The configured currency is passed in from the +// estimate API (or window.AICA_CURRENCY when rendered by the server) and +// defaults to USD. + +(function (root, factory) { + "use strict"; + var api = factory(); + if (typeof module === "object" && module.exports) { + module.exports = api; + } else { + root.AICACurrency = api; + } +})(typeof self !== "undefined" ? self : this, function () { + "use strict"; + + var DEFAULT_CURRENCY = { code: "USD", symbol: "$", decimal_places: 4 }; + + function normalize(config) { + var source = config || {}; + var code = String(source.code || DEFAULT_CURRENCY.code).trim().toUpperCase(); + var symbol = source.symbol == null ? DEFAULT_CURRENCY.symbol : String(source.symbol).trim(); + var places = Number(source.decimal_places); + if (!Number.isFinite(places)) { + places = DEFAULT_CURRENCY.decimal_places; + } + places = Math.min(Math.max(Math.trunc(places), 0), 8); + return { + code: code.slice(0, 8) || DEFAULT_CURRENCY.code, + symbol: symbol.slice(0, 8), + decimal_places: places, + }; + } + + function roundHalfUp(value, places) { + var factor = Math.pow(10, places); + return Math.round((value + Number.EPSILON) * factor) / factor; + } + + function groupThousands(value) { + var parts = value.split("."); + parts[0] = parts[0].replace(/\B(?=(\d{3})+(?!\d))/g, ","); + return parts.join("."); + } + + function formatCost(amount, config) { + var currency = normalize(config); + var value = Number(amount == null ? 0 : amount); + if (!Number.isFinite(value)) { + throw new Error("Cost amount must be finite."); + } + if (value < 0) { + throw new Error("Cost amount must be non-negative."); + } + var rounded = roundHalfUp(value, currency.decimal_places).toFixed(currency.decimal_places); + return (currency.symbol || "") + groupThousands(rounded) + " " + currency.code; + } + + function formatEstimate(estimate, config) { + var data = estimate || {}; + var currency = normalize(config || data.currency); + var tokens = Number(data.prompt_tokens) || 0; + var amount = data.estimated_cost_usd; + if (amount == null) { + amount = data.estimated_cost; + } + return ( + "Estimated: ~" + + Math.max(0, Math.trunc(tokens)).toLocaleString("en-US") + + " tokens (" + + formatCost(amount, currency) + + "; estimate)" + ); + } + + return { + DEFAULT_CURRENCY: DEFAULT_CURRENCY, + formatCost: formatCost, + formatEstimate: formatEstimate, + normalize: normalize, + }; +}); diff --git a/app/static/js/project.js b/app/static/js/project.js index af0b564..16cf8c9 100644 --- a/app/static/js/project.js +++ b/app/static/js/project.js @@ -534,9 +534,12 @@ // Pre-flight token/cost estimate (issue #104). Best-effort: failures clear // the hint rather than surfacing an error, so typing is never interrupted. function formatEstimate(estimate) { + if (window.AICACurrency && typeof window.AICACurrency.formatEstimate === "function") { + return window.AICACurrency.formatEstimate(estimate); + } var tokens = Number(estimate && estimate.prompt_tokens) || 0; var cost = Number(estimate && estimate.estimated_cost_usd) || 0; - return "Estimated: ~" + tokens.toLocaleString() + " tokens (~$" + cost.toFixed(4) + ")"; + return "Estimated: ~" + tokens.toLocaleString() + " tokens (~$" + cost.toFixed(4) + " USD; estimate)"; } var chatEstimateTimer = null; diff --git a/app/templates/base.html b/app/templates/base.html index 9cf665f..af139cf 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -72,6 +72,7 @@ + diff --git a/app/workspaces/routes.py b/app/workspaces/routes.py index cbec6b8..a044f54 100644 --- a/app/workspaces/routes.py +++ b/app/workspaces/routes.py @@ -80,7 +80,7 @@ ROLE_VIEWER, STATUS_ACTIVE, ) -from app.services import project_analysis +from app.services import project_analysis, usage_ledger from app.services.activity import record_activity from app.services.events import emit_event from app.services.exporting import export_filename, iter_export_zip @@ -1284,6 +1284,20 @@ def api_project_chat(project_id: int): ) session.updated_at = datetime.now(UTC) db.session.add(message) + usage = result.get("usage") or {} + usage_ledger.record_usage( + source=usage_ledger.SOURCE_PROJECT_CHAT, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + total_tokens=usage.get("total_tokens"), + tokens_estimated=usage.get("tokens_estimated", True), + provider=usage.get("provider"), + model=usage.get("model"), + user=current_user, + project=project, + workspace_id=project.workspace_id, + commit=False, + ) db.session.commit() return ( jsonify( @@ -1328,6 +1342,7 @@ def api_project_chat_stream(project_id: int): session.updated_at = datetime.now(UTC) db.session.commit() messages = project_analysis.build_messages(project, content, history, attachments=attachments) + provider_name, provider_model = usage_ledger.current_provider_metadata(user=current_user) def generate(): try: @@ -1339,17 +1354,32 @@ def generate(): return try: - reply = project_analysis.chat_with_project( + result = project_analysis.chat_with_project( project, content, attachments, history=history - )["analysis"] + ) except LLMProviderError as exc: yield f"data: {json.dumps({'type': 'error', 'error': str(exc)})}\n\n" return + reply = result["analysis"] + usage = result.get("usage") or {} message = ProjectMessage( project_id=project.id, session=session, role="assistant", content=reply ) db.session.add(message) + usage_ledger.record_usage( + source=usage_ledger.SOURCE_PROJECT_CHAT, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + total_tokens=usage.get("total_tokens"), + tokens_estimated=usage.get("tokens_estimated", True), + provider=usage.get("provider") or provider_name, + model=usage.get("model") or provider_model, + user=current_user, + project=project, + workspace_id=project.workspace_id, + commit=False, + ) db.session.commit() yield f"data: {json.dumps({'type': 'done', 'message': message.to_dict()})}\n\n" @@ -1415,6 +1445,21 @@ def api_project_analyze(project_id: int): if kind not in project_analysis.ANALYSIS_KINDS: return jsonify({"error": "Unsupported analysis kind."}), 400 result = project_analysis.analyze_project(project, kind) + usage = result.get("usage") or {} + provider_name, provider_model = usage_ledger.current_provider_metadata(user=current_user) + usage_ledger.record_usage( + source=usage_ledger.SOURCE_ANALYSIS, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + total_tokens=usage.get("total_tokens"), + tokens_estimated=usage.get("tokens_estimated", True), + provider=usage.get("provider") or provider_name, + model=usage.get("model") or provider_model, + user=current_user, + project=project, + workspace_id=project.workspace_id, + commit=False, + ) record_activity( project.workspace_id, EVENT_AI_ANALYSIS_RUN, diff --git a/migrations/versions/c4d5e6f7a8b9_add_usage_ledger.py b/migrations/versions/c4d5e6f7a8b9_add_usage_ledger.py new file mode 100644 index 0000000..58d6ad3 --- /dev/null +++ b/migrations/versions/c4d5e6f7a8b9_add_usage_ledger.py @@ -0,0 +1,60 @@ +"""add the central usage ledger + +Revision ID: c4d5e6f7a8b9 +Revises: e7a8b9c0d1e2 +Create Date: 2026-10-17 00:00:00.000000 + +Phase 10: central usage ledger model and migration (issue #366). The table is +append-only and stores metadata/token counts only; prompts and completions are +never written here. +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c4d5e6f7a8b9' +down_revision = 'e7a8b9c0d1e2' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'usage_records', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('workspace_id', sa.Integer(), nullable=True), + sa.Column('project_id', sa.Integer(), nullable=True), + sa.Column('conversation_id', sa.Integer(), nullable=True), + sa.Column('source', sa.String(length=40), nullable=False), + sa.Column('provider', sa.String(length=50), nullable=True), + sa.Column('model', sa.String(length=100), nullable=True), + sa.Column('prompt_tokens', sa.Integer(), nullable=False), + sa.Column('completion_tokens', sa.Integer(), nullable=False), + sa.Column('total_tokens', sa.Integer(), nullable=False), + sa.Column('tokens_estimated', sa.Boolean(), nullable=False), + sa.Column('cost_estimate', sa.Float(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['conversation_id'], ['conversations.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['project_id'], ['projects.id'], ondelete='SET NULL'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['workspace_id'], ['workspaces.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('ix_usage_records_conversation_id', 'usage_records', ['conversation_id']) + op.create_index('ix_usage_records_created_at', 'usage_records', ['created_at']) + op.create_index('ix_usage_records_project_id', 'usage_records', ['project_id']) + op.create_index('ix_usage_records_source', 'usage_records', ['source']) + op.create_index('ix_usage_records_user_id', 'usage_records', ['user_id']) + op.create_index('ix_usage_records_workspace_id', 'usage_records', ['workspace_id']) + + +def downgrade(): + op.drop_index('ix_usage_records_workspace_id', table_name='usage_records') + op.drop_index('ix_usage_records_user_id', table_name='usage_records') + op.drop_index('ix_usage_records_source', table_name='usage_records') + op.drop_index('ix_usage_records_project_id', table_name='usage_records') + op.drop_index('ix_usage_records_created_at', table_name='usage_records') + op.drop_index('ix_usage_records_conversation_id', table_name='usage_records') + op.drop_table('usage_records') diff --git a/tests/js/currency.test.js b/tests/js/currency.test.js new file mode 100644 index 0000000..03051e0 --- /dev/null +++ b/tests/js/currency.test.js @@ -0,0 +1,54 @@ +/** + * Unit tests for consistent currency/cost formatting (issue #441). + * + * Costs are estimates and must always show both the configured currency code + * and the estimate label. + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { beforeAll, describe, expect, it } from "vitest"; + +const MODULE_PATH = resolve(process.cwd(), "app/static/js/currency.js"); + +let currency; + +beforeAll(() => { + window.eval(readFileSync(MODULE_PATH, "utf8")); + currency = window.AICACurrency; + if (!currency || typeof currency.formatCost !== "function") { + throw new Error("currency.js did not expose window.AICACurrency.formatCost"); + } +}); + +describe("currency formatting", () => { + it("defaults to an explicit USD label", () => { + expect(currency.formatCost(0.0025)).toBe("$0.0025 USD"); + }); + + it("uses the configured symbol, code and precision", () => { + expect( + currency.formatCost(12.345, { code: "eur", symbol: "€", decimal_places: 2 }) + ).toBe("€12.35 EUR"); + }); + + it("groups thousands", () => { + expect( + currency.formatCost(1234567.891, { code: "USD", symbol: "$", decimal_places: 2 }) + ).toBe("$1,234,567.89 USD"); + }); + + it("rejects negative and non-finite amounts", () => { + expect(() => currency.formatCost(-0.01)).toThrow(/non-negative/); + expect(() => currency.formatCost(Number.NaN)).toThrow(/finite/); + }); + + it("labels estimates and includes the token count", () => { + const text = currency.formatEstimate({ + prompt_tokens: 1234, + estimated_cost_usd: 0.0024, + currency: { code: "USD", symbol: "$", decimal_places: 4 }, + }); + expect(text).toBe("Estimated: ~1,234 tokens ($0.0024 USD; estimate)"); + }); +}); diff --git a/tests/test_currency.py b/tests/test_currency.py new file mode 100644 index 0000000..bd61aa3 --- /dev/null +++ b/tests/test_currency.py @@ -0,0 +1,112 @@ +"""Tests for currency configuration and consistent cost formatting (issue #441).""" + +import pytest + +from app.services.currency import ( + DEFAULT_CURRENCY_CODE, + format_cost, + format_estimate, + get_currency_config, + normalize_currency, +) + + +class TestNormalizeCurrency: + def test_defaults_to_usd(self): + assert normalize_currency() == { + "code": DEFAULT_CURRENCY_CODE, + "symbol": "$", + "decimal_places": 4, + } + + def test_missing_and_malformed_values_fall_back(self): + assert normalize_currency({"code": "", "symbol": None, "decimal_places": "nope"}) == { + "code": "USD", + "symbol": "", + "decimal_places": 4, + } + + def test_precision_is_bounded(self): + assert normalize_currency({"decimal_places": -2})["decimal_places"] == 0 + assert normalize_currency({"decimal_places": 99})["decimal_places"] == 8 + + +class TestFormatCost: + def test_usd_is_explicit(self): + assert format_cost(0.0025) == "$0.0025 USD" + + def test_configured_currency_is_used(self): + currency = {"code": "eur", "symbol": "€", "decimal_places": 2} + assert format_cost(12.345, currency) == "€12.35 EUR" + + def test_rounds_half_up(self): + currency = {"code": "USD", "symbol": "$", "decimal_places": 4} + assert format_cost(0.00005, currency) == "$0.0001 USD" + + def test_thousands_are_grouped(self): + currency = {"code": "USD", "symbol": "$", "decimal_places": 2} + assert format_cost(1234567.891, currency) == "$1,234,567.89 USD" + + def test_negative_and_non_finite_are_rejected(self): + with pytest.raises(ValueError): + format_cost(-0.01) + with pytest.raises(ValueError): + format_cost(float("nan")) + + def test_format_estimate_labels_the_estimate(self): + assert format_estimate(1234, 0.0024) == ("Estimated: ~1,234 tokens ($0.0024 USD; estimate)") + + +class TestCurrencyConfig: + def test_reads_application_config(self, app): + app.config["CURRENCY_CODE"] = "GBP" + app.config["CURRENCY_SYMBOL"] = "£" + app.config["CURRENCY_DECIMAL_PLACES"] = 3 + assert get_currency_config(app) == { + "code": "GBP", + "symbol": "£", + "decimal_places": 3, + } + + def test_estimate_endpoint_returns_currency_and_display(self, client, make_user, login): + user = make_user(username="currency", email="currency@example.com") + login(email="currency@example.com") + from app.extensions import db + from app.models import Project, ProjectFile, Workspace + from app.models.project import SOURCE_ARCHIVE, STATUS_READY + + workspace = Workspace(user_id=user.id, name="Currency workspace") + db.session.add(workspace) + db.session.commit() + project = Project( + workspace_id=workspace.id, + user_id=user.id, + name="Currency project", + source=SOURCE_ARCHIVE, + status=STATUS_READY, + ) + db.session.add(project) + db.session.commit() + db.session.add( + ProjectFile( + project_id=project.id, + path="app.py", + size=10, + is_binary=False, + content="x = 1\n", + ) + ) + db.session.commit() + + response = client.post( + f"/workspaces/api/projects/{project.id}/estimate", + json={"mode": "chat", "content": "explain app.py"}, + ) + + assert response.status_code == 200 + payload = response.get_json() + assert payload["currency"]["code"] == "USD" + assert payload["cost_is_estimate"] is True + assert payload["estimated_cost_display"].endswith(" USD") + assert payload["estimated_cost_display"].startswith("$") + assert payload["estimated_cost"] == payload["estimated_cost_usd"] diff --git a/tests/test_migrations.py b/tests/test_migrations.py index dc5d3be..b3d30e2 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -126,7 +126,48 @@ class TestMigrationHead: def test_head_is_latest_revision(self): result = _run_flask(["db", "heads"], {"DATABASE_URL": "sqlite:///:memory:"}) assert result.returncode == 0, result.stderr - assert "e7a8b9c0d1e2" in (result.stdout + result.stderr) + assert "c4d5e6f7a8b9" in (result.stdout + result.stderr) + + def test_usage_records_table_upgraded(self): + expected = { + "id", + "user_id", + "workspace_id", + "project_id", + "conversation_id", + "source", + "provider", + "model", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "tokens_estimated", + "cost_estimate", + "created_at", + } + with _migration_db() as db_url, _inspect(db_url) as insp: + tables = set(insp.get_table_names()) + assert "usage_records" in tables + columns = {col["name"] for col in insp.get_columns("usage_records")} + assert columns == expected + indexes = {idx["name"] for idx in insp.get_indexes("usage_records")} + assert { + "ix_usage_records_user_id", + "ix_usage_records_workspace_id", + "ix_usage_records_project_id", + "ix_usage_records_source", + "ix_usage_records_created_at", + } <= indexes + + def test_usage_records_downgrade_removed(self): + with tempfile.TemporaryDirectory() as tmp: + db_url = f"sqlite:///{os.path.join(tmp, 'mig_ledger.db')}" + up = _run_flask(["db", "upgrade"], {"DATABASE_URL": db_url}) + assert up.returncode == 0, up.stderr + down = _run_flask(["db", "downgrade", "e7a8b9c0d1e2"], {"DATABASE_URL": db_url}) + assert down.returncode == 0, down.stderr + with _inspect(db_url) as insp: + assert "usage_records" not in set(insp.get_table_names()) def test_message_token_columns_upgraded(self): with _migration_db() as db_url, _inspect(db_url) as insp: diff --git a/tests/test_usage_ledger.py b/tests/test_usage_ledger.py new file mode 100644 index 0000000..3c18600 --- /dev/null +++ b/tests/test_usage_ledger.py @@ -0,0 +1,291 @@ +"""Tests for the central usage ledger and its source breakdown API (Phase 10). + +Covers recording/normalisation/scoping (issues #429, #432), aggregation by +source including unknown historical values (#451), and the instrumentation of +project chat and project analysis. +""" + +import pytest + +from app.extensions import db +from app.models import Project, ProjectFile, UsageRecord, Workspace +from app.models.project import SOURCE_ARCHIVE, STATUS_READY +from app.services import usage_ledger + + +def _project(make_user, login, username="ledgeruser", email="ledgeruser@example.com"): + user = make_user(username=username, email=email) + login(email=email) + workspace = Workspace(user_id=user.id, name="Ledger workspace") + db.session.add(workspace) + db.session.commit() + project = Project( + workspace_id=workspace.id, + user_id=user.id, + name="Ledger project", + source=SOURCE_ARCHIVE, + status=STATUS_READY, + ) + db.session.add(project) + db.session.commit() + db.session.add( + ProjectFile( + project_id=project.id, + path="app.py", + size=12, + is_binary=False, + content="def main(): pass\n", + ) + ) + db.session.commit() + return user, workspace, project + + +class TestRecordUsage: + def test_records_counts_scope_and_cost(self, db, make_user): + user = make_user(username="ledger", email="ledger@example.com") + workspace = Workspace(user_id=user.id, name="W") + db.session.add(workspace) + db.session.commit() + project = Project( + workspace_id=workspace.id, + user_id=user.id, + name="P", + source=SOURCE_ARCHIVE, + status=STATUS_READY, + ) + db.session.add(project) + db.session.commit() + + record = usage_ledger.record_usage( + source=" Analysis ", + prompt_tokens=100, + completion_tokens=50, + user=user, + project=project, + provider="openai", + model="gpt-4o", + tokens_estimated=True, + ) + + assert record.id is not None + assert record.source == "analysis" + assert record.workspace_id == workspace.id + assert record.project_id == project.id + assert record.user_id == user.id + assert record.prompt_tokens == 100 + assert record.completion_tokens == 50 + assert record.total_tokens == 150 + assert record.tokens_estimated is True + assert record.cost_estimate == pytest.approx(0.00075) + + def test_missing_counts_default_to_zero(self, db): + record = usage_ledger.record_usage(source="tools", provider="mock", model="mock-1") + assert record.prompt_tokens == 0 + assert record.completion_tokens == 0 + assert record.total_tokens == 0 + assert record.cost_estimate == 0.0 + + def test_unknown_model_cost_stays_unknown(self, db): + record = usage_ledger.record_usage( + source="chat", + prompt_tokens=10, + provider="openai", + model="not-a-real-model", + ) + assert record.cost_estimate is None + + def test_negative_counts_are_rejected(self, db): + with pytest.raises(ValueError): + usage_ledger.record_usage(source="chat", prompt_tokens=-1) + with pytest.raises(ValueError): + usage_ledger.record_usage(source="chat", completion_tokens=-1) + with pytest.raises(ValueError): + usage_ledger.record_usage(source="chat", cost_estimate=-0.1) + + +class TestBreakdown: + def test_groups_known_and_unknown_sources(self, db, make_user): + owner = make_user(username="owner", email="owner@example.com") + other = make_user(username="other", email="other@example.com") + + rows = [ + ("chat", 10, 5), + ("chat", 1, 1), + ("analysis", 20, 10), + ("project_chat", 3, 2), + ("review", 0, 4), + ("tools", 5, 0), + ("legacy-source", 7, 7), + ] + for source, prompt, completion in rows: + usage_ledger.record_usage( + source=source, + prompt_tokens=prompt, + completion_tokens=completion, + user=owner, + provider="mock", + model="mock-1", + ) + usage_ledger.record_usage( + source="chat", + prompt_tokens=999, + completion_tokens=999, + user=other, + provider="mock", + model="mock-1", + ) + + breakdown = usage_ledger.breakdown_by_source(user_id=owner.id) + + assert breakdown["chat"]["count"] == 2 + assert breakdown["chat"]["prompt_tokens"] == 11 + assert breakdown["chat"]["total_tokens"] == 17 + assert breakdown["analysis"]["count"] == 1 + assert breakdown["analysis"]["total_tokens"] == 30 + assert breakdown["review"]["count"] == 1 + assert breakdown["tools"]["count"] == 1 + assert breakdown["project_chat"]["count"] == 1 + assert breakdown["unknown"]["count"] == 1 + assert breakdown["unknown"]["total_tokens"] == 14 + assert breakdown["chat"]["cost_estimate"] == 0.0 + assert sum(item["count"] for item in breakdown.values()) == 7 + + def test_scope_filters_are_applied(self, db, make_user): + user = make_user(username="scope", email="scope@example.com") + workspace = Workspace(user_id=user.id, name="Scoped") + db.session.add(workspace) + db.session.commit() + project = Project( + workspace_id=workspace.id, + user_id=user.id, + name="Scoped project", + source=SOURCE_ARCHIVE, + status=STATUS_READY, + ) + db.session.add(project) + db.session.commit() + + usage_ledger.record_usage(source="chat", user=user, prompt_tokens=5) + usage_ledger.record_usage(source="analysis", user=user, project=project, prompt_tokens=7) + + by_workspace = usage_ledger.breakdown_by_source(user_id=user.id, workspace_id=workspace.id) + by_project = usage_ledger.breakdown_by_source(user_id=user.id, project_id=project.id) + + assert sum(item["count"] for item in by_workspace.values()) == 1 + assert by_workspace["analysis"]["total_tokens"] == 7 + assert sum(item["count"] for item in by_project.values()) == 1 + assert by_project["analysis"]["total_tokens"] == 7 + + +class TestBreakdownApi: + def test_requires_authentication(self, client): + response = client.get("/api/usage/breakdown") + assert response.status_code == 401 + assert response.mimetype == "application/problem+json" + + def test_returns_counts_totals_and_currency(self, client, db, make_user, login): + user = make_user(username="breakdown", email="breakdown@example.com") + login(email="breakdown@example.com") + usage_ledger.record_usage( + source="chat", user=user, provider="mock", model="mock-1", prompt_tokens=10 + ) + usage_ledger.record_usage( + source="analysis", + user=user, + provider="mock", + model="mock-1", + prompt_tokens=20, + completion_tokens=5, + ) + usage_ledger.record_usage(source="mystery", user=user, prompt_tokens=1) + + response = client.get("/api/usage/breakdown") + + assert response.status_code == 200 + payload = response.get_json() + assert payload["currency"]["code"] == "USD" + assert payload["cost_is_estimate"] is True + assert payload["by_source"]["chat"]["count"] == 1 + assert payload["by_source"]["analysis"]["total_tokens"] == 25 + assert payload["by_source"]["unknown"]["count"] == 1 + assert payload["totals"]["count"] == 3 + assert payload["totals"]["total_tokens"] == 36 + assert {item["source"] for item in payload["sources"]} >= { + "chat", + "analysis", + "unknown", + } + + def test_other_users_usage_is_not_visible(self, client, db, make_user, login): + owner = make_user(username="api-owner", email="api-owner@example.com") + usage_ledger.record_usage(source="chat", user=owner, prompt_tokens=100) + make_user(username="api-other", email="api-other@example.com") + login(email="api-other@example.com") + + payload = client.get("/api/usage/breakdown").get_json() + + assert payload["totals"]["count"] == 0 + assert all(item["count"] == 0 for item in payload["sources"]) + + +class TestProjectInstrumentation: + def test_project_chat_records_a_project_chat_row(self, client, db, make_user, login): + user, workspace, project = _project(make_user, login) + + response = client.post( + f"/workspaces/api/projects/{project.id}/chat", + json={"content": "What does app.py do?"}, + ) + + assert response.status_code == 201 + rows = UsageRecord.query.filter_by(source="project_chat").all() + assert len(rows) == 1 + row = rows[0] + assert row.user_id == user.id + assert row.workspace_id == workspace.id + assert row.project_id == project.id + assert row.total_tokens > 0 + assert row.tokens_estimated is True + assert row.provider == "mock" + assert row.model == "mock-1" + + def test_project_chat_stream_records_a_row(self, client, db, make_user, login): + _, workspace, project = _project( + make_user, login, username="streamuser", email="streamuser@example.com" + ) + + response = client.post( + f"/workspaces/api/projects/{project.id}/chat/stream", + json={"content": "Explain helper.py"}, + ) + + assert response.status_code == 200 + assert '"type": "done"' in response.get_data(as_text=True) + rows = UsageRecord.query.filter_by(source="project_chat").all() + assert len(rows) == 1 + assert rows[0].workspace_id == workspace.id + assert rows[0].project_id == project.id + assert rows[0].total_tokens > 0 + + def test_each_project_analysis_records_a_row(self, client, db, make_user, login): + user, workspace, project = _project( + make_user, login, username="analysisuser", email="analysisuser@example.com" + ) + + for kind in ("bugs", "architecture"): + response = client.post( + f"/workspaces/api/projects/{project.id}/analyze", json={"kind": kind} + ) + assert response.status_code == 200 + + rows = UsageRecord.query.filter_by(source="analysis").order_by(UsageRecord.id).all() + assert len(rows) == 2 + for row in rows: + assert row.user_id == user.id + assert row.workspace_id == workspace.id + assert row.project_id == project.id + assert row.total_tokens > 0 + assert row.tokens_estimated is True + assert row.provider == "mock" + assert row.model == "mock-1" From 3e795d26c7f9186685d2b2a43748e356ad49ac20 Mon Sep 17 00:00:00 2001 From: olaams-tech Date: Sat, 17 Oct 2026 17:46:59 +0100 Subject: [PATCH 2/2] test(chat): cover usage breakdown API and SSE reply path Keeps the PostgreSQL chat coverage gate above 85% by exercising the new ledger breakdown endpoint and the SSE reply branch of the messages API. --- tests/test_chat_api.py | 78 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/tests/test_chat_api.py b/tests/test_chat_api.py index c0e2f04..1df37c0 100644 --- a/tests/test_chat_api.py +++ b/tests/test_chat_api.py @@ -1,6 +1,7 @@ """Tests for the chat JSON API (``/api/conversations``, issue #5).""" -from app.models import Conversation, Message +from app.models import Conversation, Message, User +from app.services import usage_ledger def _register(client, username="apiuser", email="apiuser@example.com"): @@ -196,6 +197,20 @@ def test_invalid_body_is_rejected(self, client, db): assert response.status_code == 400 _problem(response) + def test_send_message_can_stream_an_sse_reply(self, client, db): + _register(client) + created = _create_conversation(client).get_json() + response = client.post( + f"/api/conversations/{created['id']}/messages", + json={"content": "Stream this reply"}, + headers={"X-CSRFToken": "ignored", "Accept": "text/event-stream"}, + ) + assert response.status_code == 200 + assert response.mimetype == "text/event-stream" + body = response.get_data(as_text=True) + assert "event: message_start" in body + assert "event: message_end" in body + def test_send_message_is_owner_scoped(self, client, db): _register(client, username="owner", email="owner@example.com") created = _create_conversation(client).get_json() @@ -210,6 +225,67 @@ def test_send_message_is_owner_scoped(self, client, db): _problem(response) +class TestUsageBreakdown: + def test_requires_authentication(self, client): + response = client.get("/api/usage/breakdown") + assert response.status_code == 401 + _problem(response) + + def test_returns_ledger_breakdown_and_usage_summary(self, client, db): + _register(client) + user = User.query.filter_by(email="apiuser@example.com").one() + usage_ledger.record_usage( + source="chat", + user=user, + provider="mock", + model="mock-1", + prompt_tokens=10, + ) + usage_ledger.record_usage( + source="analysis", + user=user, + provider="mock", + model="mock-1", + prompt_tokens=20, + completion_tokens=5, + ) + usage_ledger.record_usage(source="legacy-source", user=user, prompt_tokens=1) + + response = client.get("/api/usage/breakdown") + + assert response.status_code == 200 + payload = response.get_json() + assert payload["by_source"]["chat"]["count"] == 1 + assert payload["by_source"]["analysis"]["total_tokens"] == 25 + assert payload["by_source"]["unknown"]["count"] == 1 + assert payload["totals"]["count"] == 3 + assert payload["currency"]["code"] == "USD" + assert payload["cost_is_estimate"] is True + + conversation = Conversation(user_id=user.id, title="Usage") + db.session.add(conversation) + db.session.commit() + db.session.add( + Message( + conversation_id=conversation.id, + role="assistant", + content="reply", + prompt_tokens=4, + completion_tokens=6, + total_tokens=10, + ) + ) + db.session.commit() + + summary = client.get("/api/usage") + assert summary.status_code == 200 + assert summary.get_json() == { + "prompt_tokens": 4, + "completion_tokens": 6, + "total_tokens": 10, + } + + class TestRateLimit: def test_per_user_limit_returns_429_problem(self, client, db, app): app.config["RATE_LIMIT_CHAT_MAX"] = 1