From df41f8890fba437b2880157b6f37840122108904 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 10:39:16 -0700 Subject: [PATCH 01/94] docs: the client design, and the API the approved decks need The v1 spec's frontend and design sections predate the decks. They describe a cream palette, tabs that no longer exist and a status dot the row no longer carries. Five deck decisions also need API that does not exist: a cross-service feed, component search, a component screen, an event screen, and somewhere to send a URL we cannot read. This records what the client is built from. It settles the stack, the routes, the endpoints added and removed, the model changes, and how a status change with no provider post becomes an event. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X3FXXWHwTQ89ytonD2sZaT --- .../2026-09-03-statusboard-client-design.md | 472 ++++++++++++++++++ 1 file changed, 472 insertions(+) create mode 100644 docs/specs/2026-09-03-statusboard-client-design.md diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md new file mode 100644 index 0000000..2384dfd --- /dev/null +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -0,0 +1,472 @@ +# Statusboard client design + +**Status:** approved design, ready for an implementation plan +**Date:** 2026-09-03 + +This covers the client app, the API changes the approved designs need, and the deploy +pipeline for both images. + +Designs, approved 2026-08-23 and amended through 2026-08-28: + +- Mobile: +- Desktop: + +## 1. What this supersedes + +`docs/specs/2026-08-23-statusboard-design.md` sections 5, 6 and 7 predate the decks. +They describe a cream palette. They name the Home tabs All, Incidents and Maintenance. +They put a status dot on every row, and four tabs on a service. None of that is current. + +Sections 5, 6 and 7 are deleted, and a pointer to this document replaces them. Rewriting +them would leave two documents describing one API. That is the drift this change exists to +end. + +Sections 1 to 4 stay. Scope, architecture, the data model and the adapters are unchanged. +The columns in section 6 below are the exception. + +## 2. Work order + +1. Refresh sections 5 to 7 of the v1 spec from the decks and from this document. +2. Change the API and the models. The contract, `docs/check_docs.py` and + `api/tests/test_contract.py` move in the same commit. +3. Scaffold `app/`, with CI and deploy. + +The scaffold does not depend on steps 1 and 2. Its generated client does, so codegen +runs after step 2 lands. + +## 3. Stack + +| Concern | Choice | +| --- | --- | +| Build | Vite, React, TypeScript | +| Routing | TanStack Router | +| Server state | TanStack Query | +| API client | Orval, generated from `docs/api/openapi.yaml` | +| Styling | Tailwind v4 | +| Behaviour | Radix Primitives | +| Icons | Lucide, and only Lucide | +| Install | `vite-plugin-pwa` | +| Tests | Vitest, Testing Library, Orval's MSW handlers | +| Lint and format | Biome | + +**A Vite SPA, not SSR.** There is no offline mode, so a service worker only makes the app +installable. It caches no API response. SSR would buy link previews for shared outage +URLs. That is solvable later with a prerender, without restructuring the app. + +**Routing carries the filters.** Every filter is server-side, so every filter is a search +parameter. TanStack Router validates them against an enum in one schema. React Router +would need hand-written coercion for each one. + +**Orval generates hooks, Zod schemas and MSW handlers.** The generated client is +committed. CI regenerates it and fails on a diff. That is `makemigrations --check` for the +client: a contract change shows what it did. + +**Radix supplies behaviour only.** The decks define 26 tokens, a row anatomy and a card +catalogue. Nothing is missing but menu, tabs, popover and dialog. Those are styled from +the tokens. A component library would add a second set of colour names beside the 26. + +**There is almost no client state.** Server state is Query. Filter state is the URL. What +is left is the theme, the hidden state of Suggested, and the install banner's visit count. +All three live in local storage. No store library ships until something needs one. + +### Tokens + +The 26 tokens are one file, and nothing else writes a hex. Both decks carry an identical +`:root` core block. It is the source. + +- Neutrals `--u-0` to `--u-950`, one ultramarine ramp. +- Status, four hues with three each: raw for a fill, `-ink` for type on light, `-lift` for + type on dark. +- Semantic tokens are built from the core. Rules are built from those. + +### Auth + +The access token lives in memory. The refresh token lives in local storage. An Orval +mutator refreshes once on a 401 and retries. `POST /auth/refresh/` stays the activity +heartbeat the backend stamps `last_active_at` from. + +A strict CSP is set by the client image, because local storage is readable by script. + +**The magic link is single use.** `VerifyView` stamps `used_at` on the first success. So +the client posts that token exactly once: a mutation, `retry: false`, fired once. A query burns +the token on a StrictMode double invoke. It then tells somebody who just signed in that +their link is dead. + +The page load itself spends nothing. Only the POST does. That is what makes an email +client's prefetch safe. + +## 4. Routes + +Nine. Tabs are a search parameter, not a path. A tabbed screen is one page: the header is +shared and the body swaps, which is the decks' own rule. + +| Route | `tab` | Other search parameters | +| --- | --- | --- | +| `/` | `board`, `updates` | board: `severity`, `sort`. updates: `service`, `component`, `kind`, `phase`, `sort` | +| `/discover` | none | `q`, `severity`, `sort` | +| `/discover/add` | none | none | +| `/services/$slug` | `components`, `updates`, `about` | components: `severity`, `tracking`, `sort`. updates: `component`, `kind`, `phase`, `sort` | +| `/components/$id` | `components`, `updates` | as above, without `component` | +| `/events/$id` | `timeline`, `affects`, `about` | none | +| `/signin` | none | none | +| `/verify` | none | `token` | +| `/settings` | none | none | + +The first `tab` listed is the default. `cursor` is a search parameter on every list, +because pagination is infinite scroll. + +Components keep UUID identity. Only `Service` carries a slug. + +### A row's destination is a rule + +``` +is_overall -> /services/{service.slug} +otherwise -> /components/{id} +``` + +One function, used by every row on every screen. A component payload already carries +`is_overall` and its service, so nothing new is needed. + +### Screen to endpoint + +| Screen | Call | +| --- | --- | +| Home, board, signed in | `GET /dashboards/{uuid}/components/` | +| Home, board, signed out | `GET /catalog/components/?is_overall=true` | +| Home, updates | `GET /events/?dashboard={uuid}` | +| Discover | `GET /catalog/components/` | +| Add by URL, detected | `POST /catalog/import/` | +| Add by URL, not found | `POST /catalog/requests/` | +| Service, header and About | `GET /catalog/services/{slug}/` | +| Service, components | `GET /catalog/components/?service={slug}&is_overall=false` | +| Service, updates | `GET /events/?service={slug}` | +| Component, header | `GET /catalog/components/{uuid}/` | +| Component, components | `GET /catalog/components/?ancestor={uuid}` | +| Component, updates | `GET /events/?component={uuid}` | +| Event, header and About | `GET /events/{uuid}/` | +| Event, timeline | `GET /events/{uuid}/updates/` | +| Event, affects | `GET /catalog/components/?event={uuid}` | +| Event, affects, empty | the tab reads `Affects 0` and shows an empty state | +| Row plus | `POST /dashboards/{uuid}/components/` | +| Row menu, stop tracking | `DELETE /dashboards/{uuid}/components/{component_id}/` | +| Sign in | `POST /auth/magic-link/` | +| Verify | `POST /auth/verify/` | +| Settings | `GET /me/`, `DELETE /me/`, `POST /auth/logout/` | +| App start | `GET /meta/` | + +## 5. API surface + +15 operations become 18. + +### Removed + +| Operation | Why | +| --- | --- | +| `GET /catalog/services/` | Discover searches components. Signed-out Home lists overall components. | +| `GET /catalog/services/{slug}/components/` | `GET /catalog/components/?service=` serves it, and three other screens. | +| `GET /catalog/services/{slug}/events/` | `GET /events/?service=` serves it, and three other screens. | + +`GET /catalog/services/{slug}/` stays. The service page reads it. + +### Added + +| Operation | Parameters | +| --- | --- | +| `GET /catalog/components/` | `q`, `service`, `ancestor`, `event`, `is_overall`, `is_tracked`, `status__severity` with `lte` and `in`, `ordering` | +| `GET /catalog/components/{uuid}/` | none | +| `GET /events/` | `dashboard`, `service`, `component`, `kind`, `phase`, `ordering` | +| `GET /events/{uuid}/` | none | +| `GET /events/{uuid}/updates/` | none | +| `POST /catalog/requests/` | body `{url}` | + +`fields`, `cursor` and `page_size` stay on every operation. + +### Changed + +`GET /dashboards/{uuid}/components/` loses `event`. That parameter named the Home +Incidents and Maintenance tabs, which no longer exist. It gains +`status__severity__in`, because the Severity filter offers all six values. + +`aggregates.by_event_kind` goes, for the same reason. + +### One collection per model + +`/events/` returns `ServiceEvent`. `/events/{uuid}/updates/` returns `EventUpdate`. The +screens call the first one Updates and the second one Timeline. The API names the model +and `/meta/` publishes the labels. + +### Phase is a declared filter + +`phase` accepts `open` and `closed`. It draws `CLOSED_PHASES` from `status/choices.py`, so +the client never restates which phases are terminal. + +### A tab badge is the collection's total + +`Updates 6` is `aggregates.total` of the same `/events/` query the tab renders. No stored +count, so a badge cannot disagree with the list under it. `Components 42` and +`Components 0` stay annotations, `component_count` and `descendant_count`. A component is +a group when `descendant_count` is above zero, so no flag records it. + +### Discover is one list + +Every trackable thing is a `ServiceComponent`. A service's rollup is one of them. So +Discover searches components only, and tracking a whole service is the plus on the row +named after it. There are no tabs and no service search. + +Discover searches every component, rollups included. Only signed-out Home narrows to +`is_overall=true`, because a board is one row per service. + +`Service.component_count` excludes the overall component, and a service's Components tab +passes `is_overall=false`. The header's plus tracks the whole service, so the rollup is +reachable without being a row on its own page. + +**A component list is the whole subtree, never one level.** A service's tab lists every +component it has. A component's tab lists every descendant. The two screens are the same +screen at a different root, so they cannot count differently. + +That is why the parameter is `ancestor` and not `parent`. A self-FK lookup returns one +level, and `parent` would name a query this does not run. + +### Ordering + +| List | Default | +| --- | --- | +| `GET /catalog/components/` with no `q` | `suggested` | +| `GET /catalog/components/?service=` | `status_page_order` | +| `GET /catalog/components/` with `q` | `SearchRank` | +| `GET /events/` | `-starts_at` | + +Every one of them is labelled Smart in the interface. + +`suggested` is `(-is_featured, severity_now, -watcher_count, name)`. Severity sits ahead of +popularity, the same as the service sort it replaces. A middling component that is broken +now beats a popular one that is fine. + +There is one sort control and one label, **Smart**, on every list. With a `q` it ranks by +`SearchRank`. Without one it is `suggested`. A separate "Best match" would name a ranking +that does not exist until you type. + +## 6. Model changes + +### New model + +`catalog.ServiceRequest`, for "Send this URL to us". + +| Field | Note | +| --- | --- | +| `url` | normalised, unique. The thing an admin triages. | +| `request_count` | incremented on every post | +| `last_requested_at` | | + +`BaseModel` carries the rest. `created_at` is the first request. `created_by` is the first +requester, null when signed out. There is no `requested_by`: that is what `created_by` is. + +The endpoint upserts and increments, and always answers `202`. Asking twice cannot reveal +whether we hold a URL. It is throttled per IP, so one person cannot inflate the count. + +There is no `state`. Nothing closes a request yet, so the column would have no writer. It +arrives with the workflow that moves it. + +One row per URL, not one per request. A row per request would answer one question twice +once a URL was triaged. + +`request_count` is not the denormalisation the v1 spec bans. `Poller.last_error` was banned +for copying `PollRun.error`, which can disagree with its source. Here there is no source. +The counter is the record. + +### New columns + +| Column | Type | Written by | +| --- | --- | --- | +| `ServiceComponent.is_featured` | bool | admin | +| `ServiceComponent.watcher_count` | int | recomputed on `DashboardItem` create and delete | +| `ServiceComponent.ancestor_ids` | `ArrayField(UUIDField)`, GIN index | reconcile | +| `ServiceComponent.search_document` | `SearchVectorField`, GIN index | reconcile | +| `ServiceEvent.detected_by` | `provider` or `system` | set when the event opens, never changed | +| `EventUpdate.source` | `provider` or `system` | the adapter, or reconcile | + +### Removed columns + +`Service.is_featured` and `Service.watcher_count`. Both were keys of the service +suggestion sort, and that list is gone. + +`Service.description`. The About tab shows Website, Status page and Provider. Nothing +renders the description. So the column goes, with the adapter return value that filled it. +The v1 spec's sentence about refreshing it goes too. + +The recompute hook and the nightly reconciliation task do not disappear. They move to +`ServiceComponent`, and get simpler. `DashboardItem` points straight at a component, so +the count is `COUNT(DISTINCT owner_id)` on one indexed column. The service version needed +a distinct count across a join. + +### Changed + +| What | From | To | +| --- | --- | --- | +| `ServiceEvent.external_id` | required, unique with `service` | nullable. The unique constraint becomes partial, `WHERE external_id IS NOT NULL`. | +| `ServiceEvent.title` | always the provider's | generated when we open the event, replaced when a provider claims it | +| `EventUpdate.body` | always the provider's | generated for a `system` update | +| `Service.component_count` | every component | every component except the overall one | +| `ServiceComponent.child_count` | direct children | renamed `descendant_count`, the whole subtree | + +### Admin + +`ServiceComponent.is_featured` is how a service is featured, ticked on its overall +component. So the component admin gains that column, a list filter and a bulk action. The +service admin edits its overall component's flag inline, because "feature this service" is +the question an admin actually has. + +`ServiceComponent.watcher_count` joins the same list, read-only. + +`catalog.ServiceRequest` is registered, ordered by `request_count`. That list is the demand +signal for what the catalog is missing. + +### Choices + +`IncidentPhase` gains `DETECTED = "detected", "Detected"` as its first value. It is open, +so `CLOSED_PHASES` is unchanged. `/meta/` publishes the label, so the client renders it +with no new code. + +`EventUpdateSource` is new: `provider`, `system`. + +### Unchanged + +`User`, `Dashboard`, `DashboardItem`, `StatusPage`, `Poller`, `PollRun`, +`ComponentStatus`, and every other field on `Service`. + +`Service.slug`, `logo` and `homepage_url` stay. Routes, the row's mark and the About tab +read them. + +## 7. A status change is an event + +The Updates feed is one table. When a provider explains an outage we store their event. +When they do not, we write our own. The client never learns there were two cases. + +**Opening.** A component drops below operational and no provider event covers it. Reconcile +opens a `ServiceEvent` with `detected_by = system`, `kind = incident`, `phase = detected`, +and no `external_id`. It writes one `EventUpdate` with `source = system`. + +**While open.** Every severity transition of an affected component writes another +`EventUpdate` with `source = system`. A `ComponentStatus` transition and a provider's post +are different facts, so both appearing near one moment is correct. + +**Claiming.** A provider event claims an open unclaimed one rather than making a second. +`external_id` is filled in, the title becomes theirs, and their updates append with +`source = provider`. One outage is one card, with more updates on it. + +A candidate must be open, on the same service, with intersecting components. Its +`starts_at` must be no earlier than the system event's, less `EVENT_CLAIM_WINDOW`. The +nearest `starts_at` wins when several match. + +`EVENT_CLAIM_WINDOW` defaults to one hour, in `defaults.py`. Providers backdate `starts_at` +to when the incident really began, which is before our poll saw it. The default interval is +300s, so a short window would miss a backdated post. A system event stays open only while +the component is down. The far end is already bounded by the outage itself. + +`detected_by` is never rewritten by a claim. That is why it exists as a column. +`external_id IS NULL` cannot answer "did we find this first", because a claim fills it in. + +| `detected_by` | `external_id` | Means | +| --- | --- | --- | +| `system` | null | we found it and nobody explained it | +| `system` | set | we found it first and the provider caught up | +| `provider` | set | they posted before our poll saw the change | + +**Closing.** The component returns to operational. Reconcile writes the closing +`EventUpdate`, sets `ends_at`, and moves `phase` to `resolved`. + +A provider may post their own resolution near the same moment. Both are true, so both +appear. A timeline is ordered by `posted_at` and nothing prefers an author. The card shows +whichever landed last. + +**Archiving also closes.** Reconcile sets `archived_at` when a provider stops publishing a +component. We can no longer see it recover, so an open system event on it could never +close. Archiving writes the closing update, sets `ends_at` to `archived_at`, and resolves +the phase. The update says the component is no longer published. + +An event that cannot close leaves a permanently red row on somebody's board. + +Claiming is safer than deleting. Nothing is destroyed, so a wrong match leaves an event +with one stray first update rather than losing data. + +`ComponentStatus` stays the truth. A system event is a projection of it, written by one +writer and rebuildable from nothing. That is what keeps it from being a second answer to +the same question. + +## 8. Search + +`q` matches a component's full path. Searching `twilio` finds `SMS`, because Twilio is in +its path. Searching `twilio sms` finds it too. + +`ServiceComponent.search_document` is a `SearchVectorField` with a GIN index, written in +the reconcile pass that already sets names and parents. A path built by walking `parent` at +query time cannot use an index. + +| Weight | Source | +| --- | --- | +| `A` | the component's own name | +| `B` | its ancestors' names, read from `ancestor_ids` | +| `C` | its service's name | + +So `twilio` ranks the Twilio rollup first, then top-level components, then leaves. The +ranking solves most of the flooding a popular service would cause. `q` becomes a +`SearchQuery` in websearch mode, so several words are an AND, ordered by `SearchRank`. + +**Renaming or reparenting rewrites a whole subtree.** `ancestor_ids` and `search_document` +both carry ancestry, so both change. Providers do move components, so that rewrite belongs +inside the reconcile pass. This is the one place the design fans out. + +One column pays for three readers: the descendant query, the breadcrumb, and this weight. + +The service `search_fields` go with the service list. + +## 9. Settings + +The Refresh row is removed. It rendered `meta.poll_interval_seconds` read-only, and it +read as a control over something a user cannot set. Settings holds Theme, Suggested +services and Account. + +`/meta/` stays. It publishes the severity labels and every other enum. + +## 10. Build and test + +`app/` builds its own image from its own directory, matching `api/`. + +CI builds that image, then runs Biome, `tsc`, Vitest and the Orval drift check inside it. +What CI proves is what ships, which is the rule the API already follows. + +Tests use Orval's generated MSW handlers, so a fixture cannot describe a response the +contract does not. + +**Configuration is read at runtime, not baked in.** A Vite build inlines `VITE_` values, +which would make the image environment-specific. The container entrypoint writes +`config.js` from the environment at start, the way `api/entrypoint.sh` picks its process. +One image, any environment. + +## 11. Deploy + +`release.yml` gains an `app` job beside `api`: same tags, same cosign signing. The deploy +job waits on both. + +A push to `main` that passes CI pushes `edge` and `sha-` for both images. + +In the deployment repository: + +- the four API services take `${API_TAG:-latest}` +- a new `app` service takes `${APP_TAG:-latest}`, routed by Traefik on `statusboard.dev` +- `API_TAG` and `APP_TAG` join `.env.example` + +A deploy during development is then `APP_TAG=edge` on the server and `just deploy`. Unset +it to return to tag-gated releases. `just release` is unchanged. + +`just` gains a recipe for the client's dev server. `just dev` runs the API, the poller and +the client together. `just check` gains the client image's checks. + +## 12. Local development + +`bin/worktree-env.py` already reserves `CLIENT_PORT` and points `CORS_ALLOWED_ORIGINS` and +`CSRF_TRUSTED_ORIGINS` at `.localhost:`. Vite binds that port, and the +API already trusts that origin. That file needs no change. + +`APP_URL` and `APP_MAGIC_LINK_PATH` in `api/.env.local` gain values, so a sign-in link +reaches a page that serves it. From 661139e20975554ad11d584812a41e737d610bff Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 10:49:42 -0700 Subject: [PATCH 02/94] docs: the plan for the API the client needs Fourteen tasks: retire the v1 spec's client sections, add the columns Discover and the component screen need, write the events a provider never published, serve the six new operations, then delete the three they replace. The spec also claimed Service.watcher_count was a column to remove. Migration catalog/0002 already dropped it, and an annotation replaced it, so a component repeats neither. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01X3FXXWHwTQ89ytonD2sZaT --- .../2026-09-03-statusboard-client-design.md | 20 +- .../plans/2026-09-03-statusboard-api.md | 3200 +++++++++++++++++ 2 files changed, 3213 insertions(+), 7 deletions(-) create mode 100644 docs/superpowers/plans/2026-09-03-statusboard-api.md diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md index 2384dfd..e90270c 100644 --- a/docs/specs/2026-09-03-statusboard-client-design.md +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -279,7 +279,6 @@ The counter is the record. | Column | Type | Written by | | --- | --- | --- | | `ServiceComponent.is_featured` | bool | admin | -| `ServiceComponent.watcher_count` | int | recomputed on `DashboardItem` create and delete | | `ServiceComponent.ancestor_ids` | `ArrayField(UUIDField)`, GIN index | reconcile | | `ServiceComponent.search_document` | `SearchVectorField`, GIN index | reconcile | | `ServiceEvent.detected_by` | `provider` or `system` | set when the event opens, never changed | @@ -287,17 +286,24 @@ The counter is the record. ### Removed columns -`Service.is_featured` and `Service.watcher_count`. Both were keys of the service -suggestion sort, and that list is gone. +`Service.is_featured`. It was the first key of the service suggestion sort, and that list +is gone. `ServiceFilter` goes with it. `Service.description`. The About tab shows Website, Status page and Provider. Nothing renders the description. So the column goes, with the adapter return value that filled it. The v1 spec's sentence about refreshing it goes too. -The recompute hook and the nightly reconciliation task do not disappear. They move to -`ServiceComponent`, and get simpler. `DashboardItem` points straight at a component, so -the count is `COUNT(DISTINCT owner_id)` on one indexed column. The service version needed -a distinct count across a join. +`catalog.queries.WATCHER_COUNT` goes too. It annotated a service, and only the service +sort read it. + +**A component's watcher count is an annotation, not a column.** `Service.watcher_count` +was a column a signal kept true, and four write paths never reached the signal. Migration +`catalog/0002` dropped it for that reason. A component repeats neither the column nor the +signal. + +`COMPONENT_WATCHER_COUNT` is `Count("boards__owner", distinct=True)`, in +`catalog/queries.py`. `DashboardItem` points straight at a component, so it is one join. +The service version needed three. ### Changed diff --git a/docs/superpowers/plans/2026-09-03-statusboard-api.md b/docs/superpowers/plans/2026-09-03-statusboard-api.md new file mode 100644 index 0000000..401452f --- /dev/null +++ b/docs/superpowers/plans/2026-09-03-statusboard-api.md @@ -0,0 +1,3200 @@ +# Statusboard API Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Serve the six endpoints the approved designs need, and delete the three they replaced. + +**Architecture:** Every trackable thing is a `ServiceComponent`, so one component collection +replaces the service list and both nested service routes. A severity change nobody explains +becomes a `ServiceEvent` we write ourselves, so the Updates feed reads one table. Ancestry +moves into a stored array, which serves descendant queries, breadcrumbs and path search at once. + +**Tech Stack:** Django 5, DRF, django-filter, drf-spectacular, Postgres 17 full-text search, +pytest, factory_boy. + +**Spec:** `docs/specs/2026-09-03-statusboard-client-design.md` + +## Global Constraints + +- Python 3.14. Primary keys are `uuid.uuid7`, which does not exist earlier. +- Every model extends `common.models.BaseModel`: UUID pk, `created_at`, `updated_at`, + `created_by`, `updated_by`, `ordering = ["-created_at"]`. +- Every behaviour change carries a test. The suite runs with no network: `tests/conftest.py` + blocks `socket.socket.connect`. +- Settings split by who named them. Django or a package named it, `api/settings.py`. We named + it, `api/defaults.py`. +- Comments and docstrings follow ASD-STE100: one idea per sentence, under 20 words, active + voice, present tense, no em dashes. A comment says why, never what the line already says. +- A comment on a test says what would break if the assertion failed. +- One definition, many readers. A filter, a threshold or a set of rules lives in one place. +- An invariant the code depends on belongs in a database constraint. +- Commits use a lowercase conventional prefix, then a declarative clause. The body says why. +- `docs/api/openapi.yaml` is the contract. `api/tests/test_contract.py` fails when the code + and the contract disagree, in either direction. Both move in the same commit. +- `docs/check_docs.py` reads the field-level mermaid ER diagram in section 3 of + `docs/specs/2026-08-23-statusboard-design.md`. Every new column lands there too. +- Run `just test` for the suite and `just lint` before every commit. + +--- + +## File Structure + +**Created** + +| File | Responsibility | +| --- | --- | +| `api/catalog/views_components.py` | The component collection and its detail | +| `api/status/views.py` | The event feed, an event, and its update log | +| `api/status/filters.py` | What a caller narrows `/events/` by | +| `api/polling/system_events.py` | Opening, updating, closing and claiming a system event | +| `api/tests/test_component_api.py` | The component collection | +| `api/tests/test_event_api.py` | The event feed and an event | +| `api/tests/test_system_events.py` | The reconcile writer | +| `api/tests/test_service_requests.py` | `POST /catalog/requests/` | + +**Modified** + +| File | Change | +| --- | --- | +| `api/catalog/models.py` | `ServiceComponent.is_featured`, `ancestor_ids`, `search_document`, `descendant_count`. `Service.is_featured` and `description` go. | +| `api/catalog/queries.py` | `COMPONENT_WATCHER_COUNT` in, `WATCHER_COUNT` out | +| `api/catalog/filters.py` | `ComponentFilter` grows. `ServiceFilter` goes. | +| `api/catalog/serializers.py` | `ComponentSerializer.descendant_count`. `Service.description` out. | +| `api/catalog/views.py` | `ServiceViewSet` becomes retrieve only | +| `api/catalog/urls.py` | The component routes and the requests route | +| `api/catalog/admin.py` | Featuring, watchers, service requests | +| `api/status/models.py` | `ServiceEvent.detected_by`, nullable `external_id`, `EventUpdate.source` | +| `api/status/choices.py` | `IncidentPhase.DETECTED`, `EventUpdateSource` | +| `api/status/serializers.py` | `ServiceEventSerializer` grows a detail shape | +| `api/polling/reconcile.py` | Calls the system-event writer | +| `api/dashboards/filters.py` | `event` out, `status__severity__in` in | +| `api/common/aggregates.py` | `by_event_kind` out | +| `api/tests/factories.py` | `watchers` reads the component annotation | +| `docs/api/openapi.yaml` | Three operations out, six in | +| `docs/specs/2026-08-23-statusboard-design.md` | Sections 5 to 7 out, ER diagram updated | + +--- + +## Task 1: Retire the v1 spec's client sections + +**Files:** +- Modify: `docs/specs/2026-08-23-statusboard-design.md` + +**Interfaces:** +- Consumes: nothing +- Produces: nothing. This is a documentation change with no code behind it. + +Sections 5, 6 and 7 describe a cream palette, tabs that no longer exist and a status dot the +row no longer carries. Rewriting them would leave two documents describing one API. + +- [ ] **Step 1: Confirm the checker does not read those sections** + +Run: `grep -n 'findall\|spec\[' docs/check_docs.py` + +Expected: it reads `re.findall(r"```mermaid\n(.*?)```", spec, re.DOTALL)[1]`, the second +mermaid block, which is in section 3. Sections 5 to 7 hold no mermaid block. + +- [ ] **Step 2: Run the checker before touching anything** + +Run: `docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS. This is the baseline the next step must not move. + +- [ ] **Step 3: Delete sections 5, 6 and 7 and leave a pointer** + +Delete everything from the line `## 5. API surface` up to but not including +`## 8. Testing, infra, deploy`. Put this in their place: + +```markdown +## 5. The API, the client and the design system + +Those three sections are now +[`2026-09-03-statusboard-client-design.md`](2026-09-03-statusboard-client-design.md). + +They described a palette, tabs and a row that the approved decks replaced on 2026-08-28. +Keeping a second account of one API is how two answers to the same question drift apart. +``` + +Renumber `## 8. Testing, infra, deploy` to `## 6. Testing, infra, deploy`. + +- [ ] **Step 4: Run the checker again** + +Run: `docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS, unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add docs/specs/2026-08-23-statusboard-design.md +git commit -m "docs: one account of the API, not two + +The v1 spec's API, frontend and design sections predate the approved +decks. They name tabs that no longer exist and a palette that was +replaced. A pointer to the client design replaces them." +``` + +--- + +## Task 2: A component can be featured, and counts its own watchers + +**Files:** +- Modify: `api/catalog/models.py` +- Modify: `api/catalog/queries.py` +- Modify: `api/tests/factories.py` +- Create: `api/catalog/migrations/0003_component_is_featured.py` (generated) +- Test: `api/tests/test_catalog_models.py` + +**Interfaces:** +- Consumes: `catalog.queries.related_count`, `tests.factories.ComponentFactory`, `track` +- Produces: + - `ServiceComponent.is_featured: bool` + - `catalog.queries.COMPONENT_WATCHER_COUNT`, a `Count` expression. Annotate it as + `watcher_count` on a `ServiceComponent` queryset. + - `tests.factories.watchers(component)`, replacing the `Service` version, returns `int` + +`Service.watcher_count` was a column a signal kept true, and four write paths never reached +the signal. Migration `catalog/0002` dropped it. A component repeats neither. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_catalog_models.py`: + +```python +def test_a_components_watcher_count_is_distinct_users(db): + # Two boards holding one component is two watchers. One person + # holding it twice is impossible, and one holding a sibling too is + # still one watcher of this component. + component = ComponentFactory() + sibling = ComponentFactory(service=component.service) + first = UserFactory() + track(component, user=first) + track(sibling, user=first) + track(component, user=UserFactory()) + + assert watchers(component) == 2 + assert watchers(sibling) == 1 +``` + +Add to `api/tests/factories.py`, replacing the `Service` version of `watchers`: + +```python +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = settings.AUTH_USER_MODEL + + email = factory.Sequence(lambda n: f"watcher{n}@example.test") + + +def watchers(component): + """How many people track it, counted the way the app counts.""" + from catalog.models import ServiceComponent + from catalog.queries import COMPONENT_WATCHER_COUNT + + return ( + ServiceComponent.objects.annotate(n=COMPONENT_WATCHER_COUNT) + .get(pk=component.pk) + .n + ) +``` + +`UserFactory` needs `from django.conf import settings` and +`django.contrib.auth.get_user_model()`. Use `model = get_user_model()` rather than the +settings string, because factory_boy wants the class. + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `cd api && uv run pytest tests/test_catalog_models.py::test_a_components_watcher_count_is_distinct_users -v` +Expected: FAIL with `ImportError: cannot import name 'COMPONENT_WATCHER_COUNT'`. + +- [ ] **Step 3: Add the annotation and the column** + +In `api/catalog/queries.py`, beside `WATCHER_COUNT`: + +```python +# Who tracks one component. `DashboardItem` points straight at it, so +# this is one join. The service version needed three, and only the +# service sort read it. +COMPONENT_WATCHER_COUNT = Count("boards__owner", distinct=True) +``` + +In `api/catalog/models.py`, on `ServiceComponent`, beside `is_overall`: + +```python + # Ticked on the overall component to feature a whole service. It is + # the first key of the suggested sort, so on day one it is the + # whole of it: every watcher count starts at zero. + is_featured = models.BooleanField(verbose_name="Featured", default=False) +``` + +- [ ] **Step 4: Make the migration and run the test** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n component_is_featured +uv run pytest tests/test_catalog_models.py::test_a_components_watcher_count_is_distinct_users -v +``` +Expected: PASS. + +- [ ] **Step 5: Fix the callers the rename broke** + +Run: `cd api && uv run pytest tests/ -x -q` + +`tests/test_board_api.py` calls `watchers(service)`. Change each call to name the component +the test tracked. Expected after: the whole suite passes. + +- [ ] **Step 6: Add the column to the ER diagram** + +In `docs/specs/2026-08-23-statusboard-design.md`, in the second mermaid block, inside +`ServiceComponent { ... }`: + +``` + bool is_featured "first key of the suggested sort" +``` + +- [ ] **Step 7: Commit** + +```bash +git add api/catalog/models.py api/catalog/queries.py api/catalog/migrations api/tests docs/specs +git commit -m "feat: a component is featured and counts its own watchers + +Discover searches components, so the suggested sort needs both keys on +a component. The watcher count is an annotation, not a column: the +service had one, and four write paths never reached the signal that +kept it true." +``` + +--- + +## Task 3: Ancestry is a stored array + +**Files:** +- Modify: `api/catalog/models.py` +- Modify: `api/catalog/serializers.py` +- Modify: `api/polling/reconcile.py` +- Create: `api/catalog/migrations/0004_component_ancestor_ids.py` (generated) +- Test: `api/tests/test_catalog_models.py`, `api/tests/test_reconcile.py` + +**Interfaces:** +- Consumes: `ServiceComponent.parent`, `polling.reconcile._upsert_components` +- Produces: + - `ServiceComponent.ancestor_ids: list[UUID]`, top down, the root first, excluding self + - `ServiceComponent.descendant_count() -> int`, replacing `child_count()` + - `polling.reconcile.rebuild_ancestry(service) -> None` + +A component's Components tab lists every descendant, not one level. A self-FK lookup returns +one level, so the query needs a stored path. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_catalog_models.py`: + +```python +def test_ancestor_ids_are_stored_top_down(db): + # A descendant query reads this array. Walking `parent` per row + # cannot use an index, and `for_display` only selects two levels up. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + + leaf.refresh_from_db() + assert leaf.ancestor_ids == [top.id, middle.id] + + # Every descendant, not the direct children. `top` has two below it. + assert ServiceComponent.objects.filter(ancestor_ids__contains=[top.id]).count() == 2 +``` + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `cd api && uv run pytest tests/test_catalog_models.py::test_ancestor_ids_are_stored_top_down -v` +Expected: FAIL with `ImportError: cannot import name 'rebuild_ancestry'`. + +- [ ] **Step 3: Add the column** + +In `api/catalog/models.py`, add the import and the field: + +```python +from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.indexes import GinIndex +``` + +On `ServiceComponent`, beside `parent`: + +```python + # The chain above this one, root first, written by reconcile. A + # component's Components tab lists every descendant, and a self-FK + # lookup returns one level. Walking the chain per row cannot use an + # index either. + ancestor_ids = ArrayField( + models.UUIDField(), default=list, blank=True, editable=False + ) +``` + +In `class Meta`, add to `indexes`: + +```python + GinIndex(fields=["ancestor_ids"], name="components_by_ancestor"), +``` + +`ServiceComponent.Meta` has no `indexes` list yet. Add one beside `constraints`. + +- [ ] **Step 4: Replace child_count with descendant_count** + +In `api/catalog/models.py`, replace the `child_count` method: + +```python + def descendant_count(self): + """How many components sit anywhere under this one.""" + if self.is_overall: + return 0 + prepared = getattr(self, "_descendant_count", None) + if prepared is None: + return ServiceComponent.objects.filter(ancestor_ids__contains=[self.pk]).count() + return prepared +``` + +In `ServiceComponentQuerySet.for_display`, replace the `_child_count` annotation: + +```python + _descendant_count=related_count( + self.model.objects, "ancestor_ids__contains", ref="pk" + ), +``` + +`related_count` builds `filter(**{group_by: OuterRef(ref)})`, so `ancestor_ids__contains` +needs the ref wrapped in a list. Write the subquery directly instead: + +```python + _descendant_count=Coalesce( + Subquery( + self.model.objects.filter( + ancestor_ids__contains=Func( + OuterRef("pk"), function="ARRAY", template="ARRAY[%(expressions)s]" + ) + ) + .order_by() + .values("service") + .annotate(total=models.Count("pk")) + .values("total"), + output_field=models.IntegerField(), + ), + 0, + ), +``` + +Import `Coalesce`, `Func`, `Subquery` from `django.db.models` and +`django.db.models.functions`. + +In `api/catalog/serializers.py`, rename the field on `ComponentSerializer`: + +```python + descendant_count = serializers.SerializerMethodField() +``` + +and the method, and the entry in `Meta.fields`: + +```python + @extend_schema_field(serializers.IntegerField()) + def get_descendant_count(self, row): + return row.descendant_count() +``` + +- [ ] **Step 5: Write the rebuild in reconcile** + +In `api/polling/reconcile.py`: + +```python +def rebuild_ancestry(service): + """Write every component's chain of ancestors, root first. + + A rename does not touch this. A reparent moves a whole subtree, so + the pass rewrites the service rather than one row. A service holds + hundreds of components, not millions. + """ + parents = dict( + ServiceComponent.objects.filter(service=service).values_list("id", "parent_id") + ) + chains = {} + + def chain(node): + if node in chains: + return chains[node] + parent = parents.get(node) + # A self-referencing column allows a loop. Stop rather than hang. + chains[node] = [] if parent is None or parent == node else [ + *chain(parent), + parent, + ] + return chains[node] + + rows = list(ServiceComponent.objects.filter(service=service)) + for row in rows: + row.ancestor_ids = chain(row.id) + ServiceComponent.objects.bulk_update(rows, ["ancestor_ids"]) +``` + +`chain` recurses on the parent map, not the database, so a deep tree costs one query. +The guard stops a row that points at itself. + +Call it at the end of `_upsert_components`, after the parent pass: + +```python + rebuild_ancestry(service) + return rows +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run python manage.py makemigrations catalog -n component_ancestor_ids && uv run pytest tests/ -q` +Expected: PASS. `tests/test_catalog_api.py` asserts on `child_count`; rename those +assertions to `descendant_count`. + +- [ ] **Step 7: Update the contract and the diagram** + +In `docs/api/openapi.yaml`, rename `child_count` to `descendant_count` in the `Component` +schema, and change its description: + +```yaml + descendant_count: + type: integer + description: | + How many components sit anywhere under this one. A component is a group when this + is above zero. There is no separate flag. +``` + +In the ER diagram, inside `ServiceComponent { ... }`: + +``` + uuid[] ancestor_ids "root first, written by reconcile" +``` + +`api/tests/test_contract.py::test_the_component_schema_has_exactly_the_documented_fields` +holds these together. + +- [ ] **Step 8: Commit** + +```bash +git add api docs +git commit -m "feat: a component stores the chain above it + +A component's Components tab lists every descendant. A self-FK lookup +returns one level, and walking the chain per row cannot use an index. +Reconcile writes the array, and a reparent rewrites the subtree." +``` + +--- + +## Task 4: `q` matches a component's whole path + +**Files:** +- Modify: `api/catalog/models.py` +- Modify: `api/polling/reconcile.py` +- Create: `api/catalog/migrations/0005_component_search_document.py` (generated) +- Test: `api/tests/test_catalog_models.py` + +**Interfaces:** +- Consumes: `polling.reconcile.rebuild_ancestry` +- Produces: + - `ServiceComponent.search_document: SearchVectorField` + - `polling.reconcile.rebuild_search(service) -> None` + - `ServiceComponentQuerySet.search(q)` returns the queryset filtered and annotated with + `rank`, ordered by `-rank` + +Searching `twilio` finds `SMS`, because Twilio is in its path. `twilio sms` finds it too. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_catalog_models.py`: + +```python +def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): + # The rollup's own name is an exact hit at weight A. A leaf matches + # only through its service at weight C, so it must sort below. + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=parent) + rebuild_ancestry(service) + rebuild_search(service) + + found = list(ServiceComponent.objects.search("twilio")) + assert found[0] == rollup + assert leaf in found + + # Two words are an AND across the whole path, not two searches. + assert list(ServiceComponent.objects.search("twilio sms")) == [leaf] +``` + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `cd api && uv run pytest tests/test_catalog_models.py::test_search_matches_a_components_path_and_ranks_the_rollup_first -v` +Expected: FAIL with `ImportError: cannot import name 'rebuild_search'`. + +- [ ] **Step 3: Add the column and the queryset method** + +In `api/catalog/models.py`: + +```python +from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVectorField +``` + +On `ServiceComponent`, beside `ancestor_ids`: + +```python + # The whole path, weighted, written by reconcile. Searching a + # service's name has to reach its components, and a path built by + # walking `parent` at query time cannot use an index. + search_document = SearchVectorField(null=True, editable=False) +``` + +In `Meta.indexes`: + +```python + GinIndex(fields=["search_document"], name="components_by_search"), +``` + +On `ServiceComponentQuerySet`: + +```python + def search(self, q): + """Rank a query against the stored path. + + Websearch mode, so several words are an AND. A caller ordering + by anything else drops the rank and keeps the filter. + """ + query = SearchQuery(q, search_type="websearch") + return ( + self.filter(search_document=query) + .annotate(rank=SearchRank(models.F("search_document"), query)) + .order_by("-rank") + ) +``` + +- [ ] **Step 4: Write the rebuild** + +In `api/polling/reconcile.py`: + +```python +from django.contrib.postgres.search import SearchVector +from django.db.models import Value + + +def rebuild_search(service): + """Write each component's weighted path. + + `A` is its own name, `B` its ancestors', `C` its service's. So an + exact hit on a rollup outranks a leaf that matched through its + service alone. + + Ancestor names are joined here rather than in SQL. Postgres cannot + build a vector from an array of foreign keys. + """ + rows = list( + ServiceComponent.objects.filter(service=service).select_related("service") + ) + names = {row.id: row.name for row in rows} + for row in rows: + ancestors = " ".join(names.get(a, "") for a in row.ancestor_ids) + row.search_document = ( + SearchVector(Value(row.name), weight="A") + + SearchVector(Value(ancestors), weight="B") + + SearchVector(Value(row.service.name), weight="C") + ) + ServiceComponent.objects.bulk_update(rows, ["search_document"]) +``` + +Call it in `_upsert_components`, after `rebuild_ancestry`: + +```python + rebuild_ancestry(service) + rebuild_search(service) + return rows +``` + +`rebuild_search` must run second. It reads `ancestor_ids`. + +- [ ] **Step 5: Run the test** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n component_search_document +uv run pytest tests/test_catalog_models.py -v +``` +Expected: PASS. + +- [ ] **Step 6: Prove a reparent rewrites the subtree** + +Add to `api/tests/test_reconcile.py`: + +```python +def test_a_reparent_rewrites_every_descendants_path(db): + # Ancestry and the search vector both carry the chain. A provider + # moving one node must not leave its children searchable under + # where they used to sit. + service = ServiceFactory(name="Acme") + old = ComponentFactory(service=service, name="Legacy", external_id="old") + new = ComponentFactory(service=service, name="Platform", external_id="new") + leaf = ComponentFactory(service=service, name="Queue", external_id="leaf", parent=old) + rebuild_ancestry(service) + rebuild_search(service) + + leaf.parent = new + leaf.save(update_fields=["parent"]) + rebuild_ancestry(service) + rebuild_search(service) + + leaf.refresh_from_db() + assert leaf.ancestor_ids == [new.id] + assert list(ServiceComponent.objects.search("legacy queue")) == [] + assert list(ServiceComponent.objects.search("platform queue")) == [leaf] +``` + +- [ ] **Step 7: Run it and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +Add to the ER diagram, inside `ServiceComponent { ... }`: + +``` + tsvector search_document "weighted path, written by reconcile" +``` + +```bash +git add api docs +git commit -m "feat: a search matches a component's whole path + +Searching a service's name has to reach its components. The vector is +weighted so an exact hit on a rollup outranks a leaf that matched only +through its service. A reparent rewrites the subtree." +``` + +--- + +## Task 5: The component collection + +**Files:** +- Create: `api/catalog/views_components.py` +- Modify: `api/catalog/filters.py` +- Modify: `api/catalog/urls.py` +- Modify: `docs/api/openapi.yaml` +- Test: `api/tests/test_component_api.py` + +**Interfaces:** +- Consumes: `ServiceComponent.objects.for_display`, `.search`, + `catalog.queries.COMPONENT_WATCHER_COUNT`, `status.queries.CURRENT_SEVERITY`, + `common.aggregates.StatusAggregateSet`, `catalog.serializers.ComponentSerializer` +- Produces: + - `GET /catalog/components/` named `component-list` + - `GET /catalog/components/{uuid}/` named `component-detail` + - `ComponentFilter` with `service`, `ancestor`, `event`, `is_overall`, `is_tracked`, + `status__severity`, `status__severity__lte`, `status__severity__in` + +One collection serves Discover, a service's Components tab, a component's descendants and an +event's affected list. + +- [ ] **Step 1: Write the failing tests** + +Create `api/tests/test_component_api.py`: + +```python +import pytest +from django.urls import reverse + +from polling.reconcile import rebuild_ancestry, rebuild_search +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def tree(): + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=parent) + rebuild_ancestry(service) + rebuild_search(service) + return service, rollup, parent, leaf + + +def test_the_collection_lists_every_component(client, tree): + # Discover searches all of them, rollups included. Narrowing here + # would make the signed-out board and Discover one list. + response = client.get(reverse("component-list")) + assert response.status_code == 200 + assert response.json()["aggregates"]["total"] == 3 + + +def test_is_overall_narrows_to_one_row_per_service(client, tree): + # This is the signed-out board: one row per service, not per part. + response = client.get(reverse("component-list"), {"is_overall": "true"}) + assert [r["name"] for r in response.json()["results"]] == ["Twilio"] + + +def test_ancestor_returns_every_descendant_not_one_level(client, tree): + # A component's Components tab is the same screen at a different + # root, so it cannot count differently from a service's. + _, _, parent, leaf = tree + response = client.get(reverse("component-list"), {"ancestor": str(parent.id)}) + assert [r["id"] for r in response.json()["results"]] == [str(leaf.id)] + + +def test_q_reaches_a_component_through_its_services_name(client, tree): + # Searching "twilio" must find SMS. Its own name says nothing + # about which service it belongs to. + response = client.get(reverse("component-list"), {"q": "twilio sms"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_service_narrows_to_one_services_parts(client, tree): + # A service's Components tab. The rollup is excluded there, + # because the header already carries the service's status. + service, _, _, _ = tree + response = client.get( + reverse("component-list"), {"service": service.slug, "is_overall": "false"} + ) + assert {r["name"] for r in response.json()["results"]} == { + "Programmable Messaging", + "SMS", + } + + +def test_severity_in_takes_several_values(client, tree): + # The Severity filter offers all six values at once, so a single + # exact match cannot serve it. + response = client.get(reverse("component-list"), {"status__severity__in": "0,1,2"}) + assert response.status_code == 200 + + +def test_the_detail_answers_by_uuid(client, tree): + _, _, _, leaf = tree + response = client.get(reverse("component-detail", args=[leaf.id])) + assert response.status_code == 200 + assert response.json()["name"] == "SMS" +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_component_api.py -v` +Expected: FAIL with `NoReverseMatch: Reverse for 'component-list' not found`. + +- [ ] **Step 3: Grow the filter** + +Replace `ComponentFilter` in `api/catalog/filters.py`: + +```python +class ComponentFilter(filters.FilterSet): + # `status` is not a relation. A component has a history of statuses + # and the open one is current. Same contract name, same annotation. + status__severity = filters.NumberFilter(field_name="severity_now") + status__severity__lte = filters.NumberFilter( + field_name="severity_now", lookup_expr="lte" + ) + status__severity__in = filters.BaseInFilter( + field_name="severity_now", lookup_expr="in" + ) + service = filters.CharFilter(field_name="service__slug") + # Every descendant, not one level. `parent` would name a query this + # does not run. + ancestor = filters.UUIDFilter(method="filter_ancestor") + event = filters.UUIDFilter(field_name="events__id") + is_tracked = filters.BooleanFilter(field_name="_is_tracked") + + class Meta: + model = ServiceComponent + fields = {"is_overall": ["exact"], "is_featured": ["exact"]} + + def filter_ancestor(self, queryset, name, value): + return queryset.filter(ancestor_ids__contains=[value]) +``` + +- [ ] **Step 4: Write the views** + +Create `api/catalog/views_components.py`: + +```python +"""Every read of a component, in one place. + +Discover, a service's Components tab, a component's descendants and an +event's affected list are one collection with different parameters. +Four nested routes would have been four copies of this queryset. +""" + +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from catalog.filters import ComponentFilter +from catalog.models import ServiceComponent +from catalog.queries import COMPONENT_WATCHER_COUNT +from catalog.serializers import ComponentSerializer +from common.aggregates import StatusAggregateSet +from common.filters import FieldsBackend +from common.ordering import MappedOrderingFilter +from status.queries import CURRENT_SEVERITY + + +class ComponentQueryMixin: + permission_classes = [AllowAny] + serializer_class = ComponentSerializer + queryset = ServiceComponent.objects.none() + + def get_queryset(self): + return ( + ServiceComponent.objects.for_display(self.request.user) + .annotate( + severity_now=CURRENT_SEVERITY, + watcher_count=COMPONENT_WATCHER_COUNT, + ) + .distinct() + ) + + +class ComponentListView(ComponentQueryMixin, generics.ListAPIView): + aggregate_set = StatusAggregateSet + filterset_class = ComponentFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["name", "status_page_order", "updated_at"] + # Severity ahead of popularity, the same as the service sort this + # replaces. Lower severity is worse, so ascending puts the broken + # first. + SUGGESTED = ["-is_featured", "severity_now", "-watcher_count", "name"] + ordering_map = { + "suggested": SUGGESTED, + "status__severity": ["severity_now"], + } + ordering = SUGGESTED + + def filter_queryset(self, queryset): + """A query ranks by relevance. Without one it is the suggested sort. + + One control and one label, Smart, on every list. A separate + "best match" would name a ranking that does not exist until + somebody types. + """ + q = self.request.query_params.get("q") + if q: + queryset = queryset.search(q) + return super().filter_queryset(queryset) + + +class ComponentDetailView(ComponentQueryMixin, generics.RetrieveAPIView): + filter_backends = [FieldsBackend] + lookup_field = "pk" +``` + +`search` returns an ordered queryset. `MappedOrderingFilter` applies the view's `ordering` +after it, so the rank is lost unless the caller asked for no ordering. Guard it by making +`ordering` conditional: + +```python + def get_ordering(self): + return [] if self.request.query_params.get("q") else self.ordering +``` + +DRF's `OrderingFilter` reads `view.ordering`, not a method, so set the attribute in +`initial` instead: + +```python + def initial(self, request, *args, **kwargs): + super().initial(request, *args, **kwargs) + if request.query_params.get("q"): + # `search` already ordered by rank. A default sort applied + # after it would throw the ranking away. + self.ordering = ["-rank"] +``` + +and add `"rank"` to `ordering_fields` so `MappedOrderingFilter` does not drop it. + +- [ ] **Step 5: Route them** + +In `api/catalog/urls.py`, above the router include: + +```python + path( + "catalog/components/", + ComponentListView.as_view(), + name="component-list", + ), + path( + "catalog/components//", + ComponentDetailView.as_view(), + name="component-detail", + ), +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_component_api.py -v` +Expected: PASS, all seven. + +- [ ] **Step 7: Document both operations** + +In `docs/api/openapi.yaml`, add under `paths:`: + +```yaml + /catalog/components/: + get: + tags: [catalog] + summary: Search and list components + description: | + One collection, four screens. Discover with no parameters, a service's parts with + `service`, a component's descendants with `ancestor`, an event's affected list with + `event`. + + `q` matches a component's whole path, so `twilio` finds `SMS`. Results are ranked by + relevance. Without `q` the default is `suggested`. + security: [] + parameters: + - { name: q, in: query, schema: { type: string } } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: ancestor, in: query, schema: { type: string, format: uuid }, description: Every descendant, at any depth } + - { name: event, in: query, schema: { type: string, format: uuid } } + - { name: is_overall, in: query, schema: { type: boolean } } + - { name: is_featured, in: query, schema: { type: boolean } } + - { name: is_tracked, in: query, schema: { type: boolean } } + - { name: status__severity, in: query, schema: { type: integer } } + - { name: status__severity__lte, in: query, schema: { type: integer } } + - { name: status__severity__in, in: query, schema: { type: string }, description: 'Comma separated, e.g. `0,1,2`' } + - { name: ordering, in: query, schema: { type: string, enum: [suggested, status_page_order, name, -name, status__severity, -status__severity], default: suggested } } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/StatusAggregates' } + results: { type: array, items: { $ref: '#/components/schemas/Component' } } + /catalog/components/{uuid}/: + get: + tags: [catalog] + summary: One component + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } + '404': { $ref: '#/components/responses/Error' } +``` + +- [ ] **Step 8: Run the contract test and commit** + +Run: `cd api && uv run pytest tests/test_contract.py -v && uv run pytest tests/ -q` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: one component collection, four screens + +Discover, a service's parts, a component's descendants and an event's +affected list are one query with different parameters. Four nested +routes would have been four copies of it." +``` + +--- + +## Task 6: An event records who found it, and an update records who wrote it + +**Files:** +- Modify: `api/status/choices.py` +- Modify: `api/status/models.py` +- Create: `api/status/migrations/0003_event_detected_by.py` (generated) +- Test: `api/tests/test_status_models.py` + +**Interfaces:** +- Consumes: `status.choices.IncidentPhase`, `CLOSED_PHASES` +- Produces: + - `status.choices.EventSource`, with `PROVIDER` and `SYSTEM` + - `status.choices.IncidentPhase.DETECTED` + - `ServiceEvent.detected_by: str`, `ServiceEvent.external_id` nullable + - `EventUpdate.source: str` + +`external_id IS NULL` cannot answer "did we find this first". A claim fills it in, so the +origin fact needs a column of its own. + +- [ ] **Step 1: Write the failing test** + +In `api/tests/test_status_models.py`: + +```python +def test_two_system_events_can_sit_on_one_service(db): + # Neither has a provider id. The unique key is partial, so a null + # does not collide with another null. + service = ServiceFactory() + for _ in range(2): + ServiceEvent.objects.create( + service=service, + external_id=None, + kind=EventKind.INCIDENT, + title="Degraded", + phase=IncidentPhase.DETECTED, + detected_by=EventSource.SYSTEM, + starts_at=timezone.now(), + ) + assert ServiceEvent.objects.filter(service=service).count() == 2 + + +def test_a_provider_id_is_still_unique_per_service(db): + # A second poll of the same page must update the row, never add one. + service = ServiceFactory() + fields = dict( + service=service, + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + ServiceEvent.objects.create(external_id="abc", **fields) + with pytest.raises(IntegrityError): + ServiceEvent.objects.create(external_id="abc", **fields) + + +def test_detected_is_an_open_phase(db): + # A detected event is running. Listing it as closed would hide + # every outage no provider ever wrote up. + assert IncidentPhase.DETECTED not in CLOSED_PHASES +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_status_models.py -k "system_events or provider_id_is_still or detected_is_an_open" -v` +Expected: FAIL with `ImportError: cannot import name 'EventSource'`. + +- [ ] **Step 3: Add the choices** + +In `api/status/choices.py`, add `DETECTED` as the first value of `IncidentPhase`: + +```python +class IncidentPhase(models.TextChoices): + # First, because it precedes anything a provider posts. We write it + # when a severity drops with nothing explaining it. + DETECTED = "detected", "Detected" + INVESTIGATING = "investigating", "Investigating" + IDENTIFIED = "identified", "Identified" + MONITORING = "monitoring", "Monitoring" + RESOLVED = "resolved", "Resolved" +``` + +`CLOSED_PHASES` is unchanged. `DETECTED` is open, so it is not in it. + +Add below `MaintenancePhase`: + +```python +class EventSource(models.TextChoices): + """Who produced a row: the provider's page, or this deployment. + + An event records who opened it. An update records who wrote it. A + claimed event has both, one per update. + """ + + PROVIDER = "provider", "Provider" + SYSTEM = "system", "Statusboard" +``` + +- [ ] **Step 4: Change the models** + +In `api/status/models.py`, on `ServiceEvent`: + +```python + # Null until a provider claims this event. We open events a + # provider never published, and they have no id of theirs. + external_id = models.CharField( + verbose_name="Provider ID", max_length=200, null=True, blank=True + ) + # Who opened it, never rewritten by a claim. `external_id IS NULL` + # cannot answer this: claiming fills the column in, which destroys + # the fact that we found the outage first. + detected_by = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) +``` + +Replace the constraint in `Meta`: + +```python + constraints = [ + # Partial. Two events we opened both hold null, and null + # does not collide with null in Postgres anyway. Stating + # the condition says the exemption is deliberate. + models.UniqueConstraint( + fields=["service", "external_id"], + condition=models.Q(external_id__isnull=False), + name="one_event_per_provider_id", + ) + ] +``` + +On `EventUpdate`: + +```python + # Who wrote this post. A claimed event holds both: our detection + # first, then the provider's log. + source = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) +``` + +- [ ] **Step 5: Run the tests** + +Run: +```bash +cd api && uv run python manage.py makemigrations status -n event_detected_by +uv run pytest tests/test_status_models.py -v +``` +Expected: PASS. + +- [ ] **Step 6: Update the diagram and the contract** + +In the ER diagram, inside `ServiceEvent { ... }` change `external_id` and add a line: + +``` + string external_id "provider's id, null until claimed" + enum detected_by "provider or system, never rewritten" +``` + +Inside `EventUpdate { ... }`: + +``` + enum source "provider or system" +``` + +In `docs/api/openapi.yaml`, add `detected` to the phase enum documentation under +`Meta.enums.event_phase`, and add to `EventUpdate`'s schema: + +```yaml + source: + type: string + enum: [provider, system] + description: | + Who wrote this update. `system` is a severity change we recorded ourselves. + Render the label from `/meta/` `enums.event_source`. +``` + +Add `event_source` to `Meta.enums` in the same file. + +- [ ] **Step 7: Publish the new enum** + +In `api/common/views.py`, find where `MetaView` builds `enums` and add: + +```python + "event_source": dict(EventSource.choices), +``` + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: an event records who found it + +A provider claims an event we opened, which fills in their id. That +destroys the fact that we saw the outage first, so the origin needs a +column. An update records its own author, so one card can hold both." +``` + +--- + +## Task 7: A severity change nobody explains becomes an event + +**Files:** +- Create: `api/polling/system_events.py` +- Modify: `api/polling/reconcile.py` +- Modify: `api/api/defaults.py` +- Test: `api/tests/test_system_events.py` + +**Interfaces:** +- Consumes: `status.models.ServiceEvent`, `EventUpdate`, `ComponentStatus`, + `status.choices.EventSource`, `IncidentPhase`, `Severity` +- Produces: + - `polling.system_events.SYSTEM_EVENT_MAX_SEVERITY = 2` + - `polling.system_events.reconcile_system_events(service, author) -> None` + +A provider can move a component to Degraded and never write an incident. An event-only feed +would hide it, and those spans are exposed nowhere else. + +- [ ] **Step 1: Add the setting** + +In `api/api/defaults.py`, beside the other `POLL_*` values: + +```python +# How far back a provider's event may start and still claim one we +# opened. Providers back-date `starts_at` to when an incident really +# began, which is before our poll saw it. +EVENT_CLAIM_WINDOW = timedelta( + seconds=int(os.environ.get("EVENT_CLAIM_WINDOW_SECONDS", 3600)) +) +``` + +- [ ] **Step 2: Write the failing tests** + +Create `api/tests/test_system_events.py`: + +```python +import pytest +from django.utils import timezone + +from polling.system_events import reconcile_system_events +from status.choices import EventKind, EventSource, IncidentPhase, Severity, StatusSource +from status.models import ComponentStatus, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory + +pytestmark = pytest.mark.django_db + + +def _status(component, severity): + ComponentStatus.objects.create( + component=component, + severity=severity, + source=StatusSource.PROVIDER, + started_at=timezone.now(), + ) + + +def _author(): + from django.contrib.auth import get_user_model + + return get_user_model().objects.system() + + +def test_an_unexplained_outage_opens_an_event(db): + # Without this the feed hides every outage a provider never wrote + # up, and the closed span is exposed nowhere else. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.DEGRADED) + + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.detected_by == EventSource.SYSTEM + assert event.external_id is None + assert event.phase == IncidentPhase.DETECTED + assert event.kind == EventKind.INCIDENT + assert list(event.affected_components.all()) == [component] + assert event.updates.count() == 1 + assert event.updates.first().source == EventSource.SYSTEM + + +def test_an_explained_outage_opens_nothing(db): + # The provider already told the story. A second card for one + # outage is the thing this design exists to avoid. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + provider.affected_components.set([component]) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 0 + + +def test_unknown_opens_nothing(db): + # Severity 3 is our own poll failing to read their page. Calling + # that their incident would report our fault as theirs. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.UNKNOWN) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.count() == 0 + + +def test_every_transition_while_open_writes_an_update(db): + # A card shows how the outage moved. One update at the start would + # say a major outage had been degraded the whole time. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.updates.count() == 2 + assert event.ends_at is None + + +def test_recovery_closes_the_event(db): + # An event that cannot close leaves a permanently red row on + # somebody's board. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.OPERATIONAL) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.RESOLVED + assert event.ends_at is not None + assert event.updates.count() == 2 +``` + +- [ ] **Step 3: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_system_events.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'polling.system_events'`. + +- [ ] **Step 4: Write the module** + +Create `api/polling/system_events.py`: + +```python +"""Events we open when a provider explains nothing. + +A provider can move a component to Degraded and never write an +incident. An event-only feed would hide that outage, and the closed +`ComponentStatus` span is exposed nowhere else. + +`ComponentStatus` stays the truth. An event here is a projection of it, +written by this module alone and rebuildable from nothing. That is what +keeps it from being a second answer to the same question. +""" + +from django.conf import settings +from django.utils import timezone + +from status.choices import ( + CLOSED_PHASES, + EventKind, + EventSource, + IncidentPhase, + Severity, +) +from status.models import ComponentStatus, EventUpdate, ServiceEvent + +# Worse than this and nothing is wrong. Severity 3 is our own poll +# failing to read their page, and severity 4 is a window a provider +# always announces. Neither is an outage of theirs to report. +SYSTEM_EVENT_MAX_SEVERITY = Severity.DEGRADED + + +def reconcile_system_events(service, author): + """Open, extend and close the events this deployment owns.""" + for status in _bad_statuses(service): + _open_or_extend(status, author) + _close_recovered(service, author) + + +def _bad_statuses(service): + """Open spans bad enough to be somebody's outage.""" + return ComponentStatus.objects.filter( + component__service=service, + ended_at__isnull=True, + severity__lte=SYSTEM_EVENT_MAX_SEVERITY, + ).select_related("component") + + +def _explained(component): + """Whether a live provider event already names this component.""" + return ( + ServiceEvent.objects.live() + .filter(affected_components=component, detected_by=EventSource.PROVIDER) + .exists() + ) + + +def _ours(component): + """The open event we opened for this component, if there is one.""" + return ( + ServiceEvent.objects.live() + .filter(affected_components=component, detected_by=EventSource.SYSTEM) + .first() + ) + + +def _open_or_extend(status, author): + component = status.component + if _explained(component): + return + event = _ours(component) + if event is None: + event = ServiceEvent.objects.create( + service=component.service, + external_id=None, + detected_by=EventSource.SYSTEM, + kind=EventKind.INCIDENT, + title=_title(component, status.severity), + phase=IncidentPhase.DETECTED, + starts_at=status.started_at, + created_by=author, + updated_by=author, + ) + event.affected_components.set([component]) + _record(event, status, author) + + +def _record(event, status, author): + """One update per severity span, keyed on when the span began. + + A poll that changes nothing runs this again. `get_or_create` on the + span's start is what stops a duplicate post per beat. + """ + EventUpdate.objects.get_or_create( + event=event, + posted_at=status.started_at, + defaults={ + "phase": event.phase, + "source": EventSource.SYSTEM, + "body": _title(status.component, status.severity), + "created_by": author, + "updated_by": author, + }, + ) + + +def _close_recovered(service, author): + """Close ours once the component it names is no longer bad.""" + open_events = ServiceEvent.objects.filter( + service=service, detected_by=EventSource.SYSTEM + ).exclude(phase__in=CLOSED_PHASES) + for event in open_events: + component = event.affected_components.first() + if component is None: + continue + current = ComponentStatus.objects.filter( + component=component, ended_at__isnull=True + ).first() + if current is not None and current.severity <= SYSTEM_EVENT_MAX_SEVERITY: + continue + _close(event, component, current, author) + + +def _close(event, component, current, author): + now = timezone.now() + severity = current.severity if current is not None else Severity.OPERATIONAL + EventUpdate.objects.get_or_create( + event=event, + posted_at=now, + defaults={ + "phase": IncidentPhase.RESOLVED, + "source": EventSource.SYSTEM, + "body": _title(component, severity), + "created_by": author, + "updated_by": author, + }, + ) + event.phase = IncidentPhase.RESOLVED + event.ends_at = now + event.updated_by = author + event.save(update_fields=["phase", "ends_at", "updated_by"]) + + +def _title(component, severity): + """What the card says before a provider gives it a better name.""" + return f"{component.name} {Severity(severity).label.lower()}" +``` + +`settings.EVENT_CLAIM_WINDOW` is unused here. Task 8 uses it. + +- [ ] **Step 5: Call it from reconcile** + +In `api/polling/reconcile.py`, at the end of `apply_fetch`: + +```python + _upsert_events(service, events, rows, author, run) + # After the provider's events, so a component they explained does + # not also get one of ours in the same pass. + reconcile_system_events(service, author) +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_system_events.py -v` +Expected: PASS, all five. + +- [ ] **Step 7: Run the whole suite and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: a status change with no provider post becomes an event + +A provider can move a component to Degraded and never write an +incident. An event-only feed hid those, and the closed status span was +exposed nowhere else. The client now reads one table." +``` + +--- + +## Task 8: A provider claims the event we opened + +**Files:** +- Modify: `api/polling/system_events.py` +- Modify: `api/polling/reconcile.py` +- Test: `api/tests/test_system_events.py` + +**Interfaces:** +- Consumes: `settings.EVENT_CLAIM_WINDOW`, `polling.system_events._ours` +- Produces: `polling.system_events.claim(event, author) -> bool`, true when it took one over + +One outage is one card. A provider posting ten minutes after our poll saw the change is +more updates on the same event, not a second event. + +- [ ] **Step 1: Write the failing tests** + +Append to `api/tests/test_system_events.py`: + +```python +from datetime import timedelta + +from polling.system_events import claim + + +def test_a_provider_event_claims_the_one_we_opened(db): + # One outage is one card. Two cards for one outage is what the + # claim exists to prevent. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated SMS delivery failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(minutes=10), + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is True + + ours.refresh_from_db() + assert ours.external_id == "abc" + assert ours.title == "Elevated SMS delivery failures" + # Ours records who found it, and a claim never rewrites that. + assert ours.detected_by == EventSource.SYSTEM + assert not ServiceEvent.objects.filter(pk=provider.pk).exists() + + +def test_a_stale_provider_event_claims_nothing(db): + # An incident that began a day before ours is a different outage. + # Merging them would put one card's updates on another's timeline. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Yesterday", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(days=1), + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is False + assert ServiceEvent.objects.count() == 2 + + +def test_archiving_a_component_closes_our_open_event(db): + # We can no longer watch it recover, so the event could never + # close on its own. A red row would stay on a board for good. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + component.is_archived = True + component.save(update_fields=["is_archived"]) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.RESOLVED + assert event.ends_at is not None +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_system_events.py -k "claims or archiving" -v` +Expected: FAIL with `ImportError: cannot import name 'claim'`. + +- [ ] **Step 3: Write the claim** + +In `api/polling/system_events.py`: + +```python +def claim(provider_event, author): + """Fold a provider's event into the one we opened for the same outage. + + A provider often posts minutes after our poll saw the change. Two + rows would be two cards for one outage. + + The provider's row is deleted and ours takes its id, so the fact + that we found it first survives. Deleting theirs rather than ours + is what keeps `detected_by` meaningful. + """ + if provider_event.detected_by != EventSource.PROVIDER: + return False + ours = _claimable(provider_event) + if ours is None: + return False + ours.external_id = provider_event.external_id + ours.title = provider_event.title + ours.phase = provider_event.phase + ours.ends_at = provider_event.ends_at + ours.updated_by = author + ours.save( + update_fields=["external_id", "title", "phase", "ends_at", "updated_by"] + ) + provider_event.updates.update(event=ours) + ours.affected_components.add(*provider_event.affected_components.all()) + provider_event.delete() + return True + + +def _claimable(provider_event): + """Our open event for the same outage, or nothing. + + Same service, intersecting components, and a start no earlier than + ours less the window. The nearest start wins when several match. + """ + components = list(provider_event.affected_components.all()) + if not components: + return None + floor = provider_event.starts_at + settings.EVENT_CLAIM_WINDOW + candidates = ( + ServiceEvent.objects.live() + .filter( + service=provider_event.service, + detected_by=EventSource.SYSTEM, + external_id__isnull=True, + affected_components__in=components, + starts_at__lte=floor, + ) + .distinct() + ) + return min( + candidates, + key=lambda e: abs(e.starts_at - provider_event.starts_at), + default=None, + ) +``` + +`starts_at__lte=floor` reads as "ours began no later than theirs plus the window", which is +the same rule as "theirs began no earlier than ours less the window". + +- [ ] **Step 4: Close an event whose component is archived** + +In `_close_recovered`, replace the body of the loop: + +```python + for event in open_events: + component = event.affected_components.first() + if component is None: + continue + current = ComponentStatus.objects.filter( + component=component, ended_at__isnull=True + ).first() + still_bad = current is not None and current.severity <= SYSTEM_EVENT_MAX_SEVERITY + # An archived component is one the provider stopped publishing. + # We cannot watch it recover, so the event could never close. + if still_bad and not component.is_archived: + continue + _close(event, component, current, author) +``` + +- [ ] **Step 5: Call the claim from reconcile** + +In `api/polling/reconcile.py`, at the end of `_upsert_events`'s loop body: + +```python + for update in incoming.updates: + EventUpdate.objects.get_or_create( + event=event, + posted_at=update.posted_at, + defaults={ + "phase": update.phase, + "body": update.body, + "source": EventSource.PROVIDER, + "created_by": author, + "updated_by": author, + }, + ) + # After the updates, so they move with the row if this event + # takes over one we opened. + claim(event, author) +``` + +Import `claim` and `EventSource` at the top of `reconcile.py`. + +`_upsert_events` uses `update_or_create` keyed on `(service, external_id)`. Once ours has +taken the id, the next poll finds ours and updates it in place, so `claim` returns false +and nothing happens twice. + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_system_events.py -v` +Expected: PASS, all eight. + +- [ ] **Step 7: Prove a claimed event survives the next poll** + +Append to `api/tests/test_system_events.py`: + +```python +def test_a_claimed_event_is_not_claimed_twice(db): + # The next poll finds ours by the id it took, so `update_or_create` + # updates it in place. A second claim would delete the row it just + # matched. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at, + ) + provider.affected_components.set([component]) + claim(provider, _author()) + + ours.refresh_from_db() + assert claim(ours, _author()) is False + assert ServiceEvent.objects.count() == 1 +``` + +`claim` returns false here because `ours.detected_by` is `system`, which the first guard +rejects. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format .` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: a provider's post joins the event we already opened + +Twilio's dot goes red at 14:02 and they post at 14:14. Two rows would +be two cards for one outage. Theirs is folded into ours, so the card +gains updates rather than a duplicate, and detected_by survives." +``` + +--- + +## Task 9: The event feed + +**Files:** +- Create: `api/status/views.py` +- Create: `api/status/filters.py` +- Create: `api/status/urls.py` +- Modify: `api/api/urls.py` +- Modify: `docs/api/openapi.yaml` +- Test: `api/tests/test_event_api.py` + +**Interfaces:** +- Consumes: `status.models.ServiceEvent`, `status.serializers.ServiceEventSerializer`, + `common.aggregates.EventAggregateSet`, `status.choices.CLOSED_PHASES` +- Produces: + - `GET /events/` named `event-list` + - `EventFilter` with `dashboard`, `service`, `component`, `kind`, `phase` + +One feed serves Home, a service and a component. The screens call it Updates. The API names +the model. + +- [ ] **Step 1: Write the failing tests** + +Create `api/tests/test_event_api.py`: + +```python +import pytest +from django.urls import reverse +from django.utils import timezone + +from status.choices import EventKind, IncidentPhase, MaintenancePhase +from status.models import ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +def _event(service, component, **kwargs): + fields = dict( + kind=EventKind.INCIDENT, + title="Something broke", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + fields.update(kwargs) + event = ServiceEvent.objects.create(service=service, **fields) + event.affected_components.set([component]) + return event + + +def test_the_feed_lists_events_newest_first(client): + # A feed is read from the top. Oldest first would open on an + # incident from last month. + service = ServiceFactory() + component = ComponentFactory(service=service) + old = _event(service, component, external_id="1", title="Older") + new = _event(service, component, external_id="2", title="Newer") + ServiceEvent.objects.filter(pk=new.pk).update( + starts_at=timezone.now() + timezone.timedelta(minutes=5) + ) + + results = client.get(reverse("event-list")).json()["results"] + assert [r["title"] for r in results] == ["Newer", "Older"] + + +def test_service_narrows_the_feed(client): + # A service's Updates tab. Anything from another service on it + # would be answering a question nobody asked on that page. + first = ServiceFactory() + second = ServiceFactory() + _event(first, ComponentFactory(service=first), external_id="1", title="Mine") + _event(second, ComponentFactory(service=second), external_id="2", title="Theirs") + + results = client.get(reverse("event-list"), {"service": first.slug}).json()["results"] + assert [r["title"] for r in results] == ["Mine"] + + +def test_component_narrows_the_feed(client): + # A component's Updates tab reads the same collection one level + # down, so it cannot be a different endpoint. + service = ServiceFactory() + wanted = ComponentFactory(service=service) + other = ComponentFactory(service=service) + _event(service, wanted, external_id="1", title="Wanted") + _event(service, other, external_id="2", title="Other") + + results = client.get( + reverse("event-list"), {"component": str(wanted.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Wanted"] + + +def test_phase_open_and_closed_draw_one_line(client): + # `CLOSED_PHASES` lives in status/choices.py. A client restating + # which phases are terminal is a second copy that can drift. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Open") + _event( + service, + component, + external_id="2", + title="Done", + phase=IncidentPhase.RESOLVED, + ) + _event( + service, + component, + external_id="3", + title="Finished", + kind=EventKind.MAINTENANCE, + phase=MaintenancePhase.COMPLETED, + ) + + open_titles = client.get(reverse("event-list"), {"phase": "open"}).json()["results"] + closed = client.get(reverse("event-list"), {"phase": "closed"}).json()["results"] + assert [r["title"] for r in open_titles] == ["Open"] + assert {r["title"] for r in closed} == {"Done", "Finished"} + + +def test_dashboard_narrows_to_what_you_track(client, board): + # Home's Updates tab. Everything posted across the services on + # your board, and nothing else. + tracked = ComponentFactory() + untracked = ComponentFactory() + track(tracked, user=board.owner) + _event(tracked.service, tracked, external_id="1", title="Yours") + _event(untracked.service, untracked, external_id="2", title="Not yours") + + client.force_login(board.owner) + results = client.get( + reverse("event-list"), {"dashboard": str(board.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Yours"] +``` + +`board` is the fixture `api/tests/test_board_api.py` already uses. Move it into +`api/tests/conftest.py` so both files reach it. + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_event_api.py -v` +Expected: FAIL with `NoReverseMatch: Reverse for 'event-list' not found`. + +- [ ] **Step 3: Write the filter** + +Create `api/status/filters.py`: + +```python +"""What a caller may narrow the event feed by. + +Beside the views, and apart from them. A filter is the query contract, +and it is read on its own more often than the view around it. +""" + +from django_filters import rest_framework as filters + +from status.choices import CLOSED_PHASES +from status.models import ServiceEvent + +PHASE_STATES = [("open", "Open"), ("closed", "Closed")] + + +class EventFilter(filters.FilterSet): + # Declared, not generated. The line between an open phase and a + # closed one is `CLOSED_PHASES`, and a client restating it is a + # second copy of one rule. + phase = filters.ChoiceFilter(choices=PHASE_STATES, method="filter_phase") + service = filters.CharFilter(field_name="service__slug") + component = filters.UUIDFilter(field_name="affected_components__id") + dashboard = filters.UUIDFilter(method="filter_dashboard") + + class Meta: + model = ServiceEvent + fields = {"kind": ["exact"]} + + def filter_phase(self, queryset, name, value): + if value == "closed": + return queryset.filter(phase__in=CLOSED_PHASES) + return queryset.exclude(phase__in=CLOSED_PHASES) + + def filter_dashboard(self, queryset, name, value): + """Everything posted across the services on one board. + + The board's rows are components, and an event names components. + A user reaches only their own board: the view checks the owner + before this runs. + """ + return queryset.filter( + affected_components__tracked_by__dashboard_id=value + ).distinct() +``` + +- [ ] **Step 4: Write the view** + +Create `api/status/views.py`: + +```python +"""Every read of an event, in one place. + +Home, a service and a component show the same feed at different scopes. +The screens call it Updates. This names the model, and `/meta/` +publishes the labels. +""" + +from django.shortcuts import get_object_or_404 +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from common.aggregates import EventAggregateSet +from common.filters import FieldsBackend +from common.ordering import MappedOrderingFilter +from dashboards.models import Dashboard +from status.filters import EventFilter +from status.models import ServiceEvent +from status.serializers import ServiceEventSerializer + + +class EventListView(generics.ListAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventSerializer + aggregate_set = EventAggregateSet + filterset_class = EventFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["starts_at", "ends_at"] + ordering_map = {} + ordering = ["-starts_at"] + queryset = ServiceEvent.objects.none() + + def get_queryset(self): + queryset = ServiceEvent.objects.select_related("service").prefetch_related( + "updates" + ) + board = self.request.query_params.get("dashboard") + if board: + # 404 rather than 403: someone else's board id should not + # be confirmable. The filter runs after this. + get_object_or_404(Dashboard, id=board, owner=self.request.user) + return queryset +``` + +An anonymous caller passing `dashboard` reaches `get_object_or_404` with +`owner=AnonymousUser`, which raises `TypeError`. Guard it: + +```python + if board: + if not self.request.user.is_authenticated: + raise NotAuthenticated + get_object_or_404(Dashboard, id=board, owner=self.request.user) +``` + +Import `NotAuthenticated` from `rest_framework.exceptions`. + +- [ ] **Step 5: Route it** + +Create `api/status/urls.py`: + +```python +from django.urls import path + +from status.views import EventListView + +urlpatterns = [ + path("events/", EventListView.as_view(), name="event-list"), +] +``` + +In `api/api/urls.py`, beside the other includes: + +```python + path("", include("status.urls")), +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_event_api.py -v` +Expected: PASS, all five. + +- [ ] **Step 7: Document the operation** + +In `docs/api/openapi.yaml`, add under `paths:`: + +```yaml + /events/: + get: + tags: [status] + summary: The event feed + description: | + One feed, three scopes. Home with `dashboard`, a service with `service`, a component + with `component`. + + `phase` takes `open` or `closed`. The line between them is `CLOSED_PHASES` in the + code, so a client never restates which phases are terminal. + security: [] + parameters: + - { name: dashboard, in: query, schema: { type: string, format: uuid }, description: Requires auth, and must be your own } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: component, in: query, schema: { type: string, format: uuid } } + - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] } } + - { name: phase, in: query, schema: { type: string, enum: [open, closed] } } + - { name: ordering, in: query, schema: { type: string, enum: [-starts_at, starts_at, ends_at, -ends_at], default: -starts_at } } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/EventAggregates' } + results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } +``` + +- [ ] **Step 8: Run the contract test and commit** + +Run: `cd api && uv run pytest tests/test_contract.py tests/ -q` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: one event feed for Home, a service and a component + +Three screens showed the same list at different scopes. The phase +filter draws its line from CLOSED_PHASES, so no client carries a copy +of which phases are terminal." +``` + +--- + +## Task 10: One event, and its update log + +**Files:** +- Modify: `api/status/views.py` +- Modify: `api/status/urls.py` +- Modify: `api/status/serializers.py` +- Modify: `docs/api/openapi.yaml` +- Test: `api/tests/test_event_api.py` + +**Interfaces:** +- Consumes: `status.serializers.ServiceEventSerializer`, `EventUpdateSerializer` +- Produces: + - `GET /events/{uuid}/` named `event-detail` + - `GET /events/{uuid}/updates/` named `event-updates` + - `ServiceEventDetailSerializer` with `update_count` and `affected_count` + +The event screen's header needs both counts before either tab is opened. Its Timeline is a +paged list, because a provider's log has no ceiling. + +- [ ] **Step 1: Write the failing tests** + +Append to `api/tests/test_event_api.py`: + +```python +from status.choices import EventSource +from status.models import EventUpdate + + +def test_the_detail_carries_both_tab_counts(client): + # The header draws `Timeline 3` and `Affects 2` before either tab + # is opened, so neither can wait for that tab's request. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service) + event = _event(service, first, external_id="1") + event.affected_components.add(second) + for minute in range(3): + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body=f"update {minute}", + posted_at=timezone.now(), + ) + + body = client.get(reverse("event-detail", args=[event.id])).json() + assert body["update_count"] == 3 + assert body["affected_count"] == 2 + + +def test_the_timeline_is_its_own_paged_list(client): + # A provider's log has no ceiling, so it cannot ride on the detail. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Cause found", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) + + body = client.get(reverse("event-updates", args=[event.id])).json() + assert body["aggregates"]["total"] == 1 + assert body["results"][0]["source"] == "provider" + + +def test_the_timeline_is_oldest_first(client): + # A story is read forwards. The feed is newest first because it is + # a feed; one event's log is a narrative. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + later = EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Second", + posted_at=timezone.now() + timezone.timedelta(minutes=1), + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body="First", + posted_at=timezone.now(), + ) + + results = client.get(reverse("event-updates", args=[event.id])).json()["results"] + assert [r["body"] for r in results] == ["First", "Second"] +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_event_api.py -k "detail_carries or timeline" -v` +Expected: FAIL with `NoReverseMatch: Reverse for 'event-detail' not found`. + +- [ ] **Step 3: Add the serializers** + +In `api/status/serializers.py`, add `source` to `EventUpdateSerializer`: + +```python +class EventUpdateSerializer(FieldsMixin, serializers.ModelSerializer): + class Meta: + model = EventUpdate + fields = ["phase", "body", "posted_at", "source"] +``` + +Remove the nested `updates` from `ServiceEventSerializer`. A feed row costs a title, not a +log, and the log is now its own operation: + +```python +class ServiceEventSerializer(FieldsMixin, serializers.ModelSerializer): + """One row of the feed. The log is `/events/{uuid}/updates/`.""" + + class Meta: + model = ServiceEvent + fields = [ + "id", + "kind", + "title", + "phase", + "starts_at", + "ends_at", + "detected_by", + ] + + +class ServiceEventDetailSerializer(ServiceEventSerializer): + """The event screen's header and its About tab. + + Both counts are here because the header draws both tab badges + before either tab has made a request. + """ + + update_count = serializers.SerializerMethodField() + affected_count = serializers.SerializerMethodField() + last_update_at = serializers.SerializerMethodField() + + class Meta(ServiceEventSerializer.Meta): + fields = [ + *ServiceEventSerializer.Meta.fields, + "update_count", + "affected_count", + "last_update_at", + ] + + @extend_schema_field(serializers.IntegerField()) + def get_update_count(self, event): + return event.updates.count() + + @extend_schema_field(serializers.IntegerField()) + def get_affected_count(self, event): + return event.affected_components.count() + + @extend_schema_field(serializers.DateTimeField(allow_null=True)) + def get_last_update_at(self, event): + newest = event.updates.order_by("-posted_at").first() + return newest.posted_at if newest else None +``` + +`ServiceEventSerializer` gains `detected_by`, so a card can say the outage was ours to find. + +- [ ] **Step 4: Add the views** + +In `api/status/views.py`: + +```python +class EventDetailView(generics.RetrieveAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventDetailSerializer + filter_backends = [FieldsBackend] + queryset = ServiceEvent.objects.all() + + +class EventUpdateListView(generics.ListAPIView): + """One event's log, oldest first. + + The feed is newest first because it is a feed. This is a narrative, + and a narrative is read forwards. + """ + + permission_classes = [AllowAny] + serializer_class = EventUpdateSerializer + filter_backends = [FieldsBackend] + queryset = EventUpdate.objects.none() + + def get_queryset(self): + event = get_object_or_404(ServiceEvent, pk=self.kwargs["pk"]) + return event.updates.order_by("posted_at") +``` + +`EnvelopePagination.get_ordering` appends `-created_at` as the cursor tiebreak, which +would fight `posted_at`. Give this view its own paginator: + +```python +class TimelinePagination(EnvelopePagination): + ordering = "posted_at" + tiebreak = "created_at" +``` + +Put it in `api/common/pagination.py` and set `pagination_class = TimelinePagination` on +`EventUpdateListView`. + +- [ ] **Step 5: Route them** + +In `api/status/urls.py`: + +```python + path("events//", EventDetailView.as_view(), name="event-detail"), + path( + "events//updates/", + EventUpdateListView.as_view(), + name="event-updates", + ), +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_event_api.py -v` +Expected: PASS, all eight. + +- [ ] **Step 7: Document both, and split the schema** + +In `docs/api/openapi.yaml`, remove `updates` from the `ServiceEvent` schema, add +`detected_by`, and add a `ServiceEventDetail` schema: + +```yaml + ServiceEventDetail: + allOf: + - $ref: '#/components/schemas/ServiceEvent' + - type: object + description: | + The event screen's header. Both counts are here because the header draws both tab + badges before either tab has made a request. + properties: + update_count: { type: integer } + affected_count: { type: integer } + last_update_at: { type: string, format: date-time, nullable: true } +``` + +Add the two operations: + +```yaml + /events/{uuid}/: + get: + tags: [status] + summary: One event + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/ServiceEventDetail' } } } } + '404': { $ref: '#/components/responses/Error' } + /events/{uuid}/updates/: + get: + tags: [status] + summary: One event's log, oldest first + description: Its own operation because a provider's log has no ceiling. + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/Aggregates' } + results: { type: array, items: { $ref: '#/components/schemas/EventUpdate' } } +``` + +Add `ServiceEventDetail` to `BACKED` in `docs/check_docs.py`, mapped to `ServiceEvent`, +and add `ServiceEventDetail.update_count`, `.affected_count` and `.last_update_at` to +`DERIVED`. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: an event, and its log as its own list + +The header draws both tab badges before either tab asks for anything, +so the detail carries both counts. The log is paged on its own: a +provider writes as many updates as they like." +``` + +--- + +## Task 11: Somewhere to send a URL we cannot read + +**Files:** +- Modify: `api/catalog/models.py` +- Create: `api/catalog/serializers.py` addition +- Modify: `api/catalog/views_components.py` or a new view module +- Modify: `api/catalog/urls.py` +- Modify: `api/api/defaults.py` +- Test: `api/tests/test_service_requests.py` + +**Interfaces:** +- Consumes: `common.models.BaseModel`, `catalog.models.StatusPage.normalise_url`, + `api.defaults.Throttle` +- Produces: + - `catalog.models.ServiceRequest` with `url`, `request_count`, `last_requested_at` + - `POST /catalog/requests/` named `catalog-request` + +The Add-by-URL screen's not-found state offers "Send this URL to us". `POST /catalog/import/` +stores nothing about a failed attempt, so that button has nowhere to send anything. + +- [ ] **Step 1: Write the failing tests** + +Create `api/tests/test_service_requests.py`: + +```python +import pytest +from django.urls import reverse + +from catalog.models import ServiceRequest + +pytestmark = pytest.mark.django_db + + +def test_asking_records_the_url(client): + response = client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert response.status_code == 202 + row = ServiceRequest.objects.get() + assert row.url == "https://status.fastmail.com" + assert row.request_count == 1 + + +def test_asking_twice_counts_rather_than_duplicates(client): + # State belongs to the URL, not to a request. A row per request + # would answer one question twice once a URL was triaged. + for _ in range(2): + client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert ServiceRequest.objects.count() == 1 + assert ServiceRequest.objects.get().request_count == 2 + + +def test_the_answer_is_the_same_whether_we_hold_it_or_not(client): + # Always 202. A different code would reveal which URLs are + # already in the catalog to anyone who asked. + first = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + second = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + assert first.status_code == second.status_code == 202 + assert first.content == second.content + + +def test_a_signed_in_asker_is_recorded(client, user): + # created_by carries it. There is no requested_by: that is what + # BaseModel already holds. + client.force_login(user) + client.post( + reverse("catalog-request"), + {"url": "https://status.two.example/"}, + content_type="application/json", + ) + assert ServiceRequest.objects.get().created_by == user + + +def test_a_malformed_url_is_refused(client): + response = client.post( + reverse("catalog-request"), + {"url": "not a url"}, + content_type="application/json", + ) + assert response.status_code == 400 +``` + +`user` is a fixture. Add it to `api/tests/conftest.py` if it is not already there: + +```python +@pytest.fixture +def user(db): + from django.contrib.auth import get_user_model + + return get_user_model().objects.create(email="asker@example.test") +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_service_requests.py -v` +Expected: FAIL with `ImportError: cannot import name 'ServiceRequest'`. + +- [ ] **Step 3: Add the model** + +In `api/catalog/models.py`, below `StatusPage`: + +```python +class ServiceRequest(BaseModel): + """A status page somebody asked for and we could not read. + + One row per URL, not one per request. A row per request would + answer one question twice once a URL was triaged. + + There is no `requested_by`. `BaseModel.created_by` already holds + it, null when the asker was signed out. The v1 spec deleted + `Service.added_by` for the same reason. + + There is no state column either. Nothing closes a request yet, so + it would have no writer. + """ + + url = models.URLField(unique=True) + # The demand signal. Nothing else records how often a URL was + # asked for, so this is the record rather than a copy of one. + request_count = models.PositiveIntegerField(default=1) + last_requested_at = models.DateTimeField(default=timezone.now) + + class Meta(BaseModel.Meta): + ordering = ["-request_count", "-last_requested_at"] + + def __str__(self): + return self.url +``` + +- [ ] **Step 4: Add the serializer and the view** + +In `api/catalog/serializers.py`: + +```python +class ServiceRequestSerializer(serializers.Serializer): + """The body of POST /catalog/requests/.""" + + url = serializers.URLField() +``` + +In `api/catalog/views.py`, below `CatalogImportView`: + +```python +class ServiceRequestView(APIView): + """"Send this URL to us", from the Add-by-URL not-found screen. + + An import stores nothing about an attempt that failed, so this is + where a dead end goes. Telling somebody to hunt for a better link + assumes they have not already tried. + """ + + permission_classes = [AllowAny] + # An anonymous write. Without this one person could inflate the + # demand signal the admin list is ordered by. + throttle_scope = Throttle.IMPORT + + @extend_schema( + request=ServiceRequestSerializer, + responses={ + 202: OpenApiResponse(description="Recorded."), + 400: OpenApiResponse(description="Missing or malformed url."), + }, + ) + def post(self, request): + body = ServiceRequestSerializer(data=request.data) + if not body.is_valid(): + return Response(body.errors, status=400) + url = StatusPage.normalise_url(body.validated_data["url"]) + author = request.user if request.user.is_authenticated else None + row, created = ServiceRequest.objects.get_or_create( + url=url, defaults={"created_by": author, "updated_by": author} + ) + if not created: + # F() rather than a read and a write. Two people asking at + # once would otherwise both write 2. + ServiceRequest.objects.filter(pk=row.pk).update( + request_count=F("request_count") + 1, + last_requested_at=timezone.now(), + ) + # Always 202, and always the same body. Anything else would + # tell a stranger which URLs the catalog already holds. + return Response(status=http.HTTP_202_ACCEPTED) +``` + +Import `F` from `django.db.models`, `timezone` from `django.utils`, and `ServiceRequest`, +`StatusPage`, `ServiceRequestSerializer`. + +- [ ] **Step 5: Route it** + +In `api/catalog/urls.py`, beside the import path: + +```python + path( + "catalog/requests/", + ServiceRequestView.as_view(), + name="catalog-request", + ), +``` + +- [ ] **Step 6: Run the tests** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n service_request +uv run pytest tests/test_service_requests.py -v +``` +Expected: PASS, all five. + +- [ ] **Step 7: Document it** + +In `docs/api/openapi.yaml`: + +```yaml + /catalog/requests/: + post: + tags: [catalog] + summary: Ask us for a status page we could not read + description: | + The Add-by-URL not-found screen. Always `202`, whether or not the catalog already + holds the URL, so asking cannot reveal what we have. + + One row per URL, counted. Throttled per caller. + security: [] + requestBody: + required: true + content: + application/json: + schema: { type: object, required: [url], properties: { url: { type: string, format: uri } } } + responses: + '202': { description: Recorded } + '400': { $ref: '#/components/responses/Error' } + '429': { $ref: '#/components/responses/Throttled' } +``` + +Add `ServiceRequest` to the ER diagram in the v1 spec: + +``` + ServiceRequest { + uuid id PK + url url UK "normalised" + int request_count "the demand signal" + datetime last_requested_at + } +``` + +and to `BACKED` in `docs/check_docs.py`. It returns no body, so it needs no schema entry. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && docker compose -f docker-compose.test.yml run --rm docs` +Expected: PASS. + +```bash +git add api docs +git commit -m "feat: a dead end can hand us the address + +The Add-by-URL not-found screen offers to send us a URL, and an import +stored nothing about an attempt that failed. One row per URL with a +count, so the admin list is ordered by what people actually want." +``` + +--- + +## Task 12: Delete what nothing reads + +**Files:** +- Modify: `api/catalog/views.py`, `api/catalog/urls.py`, `api/catalog/filters.py`, + `api/catalog/queries.py`, `api/catalog/models.py`, `api/catalog/serializers.py` +- Modify: `api/dashboards/filters.py` +- Modify: `api/common/aggregates.py` +- Modify: `docs/api/openapi.yaml`, `docs/specs/2026-08-23-statusboard-design.md` +- Create: `api/catalog/migrations/0007_drop_service_columns.py` (generated) +- Test: `api/tests/test_catalog_api.py`, `api/tests/test_contract.py` + +**Interfaces:** +- Consumes: everything the previous tasks produced. This one only removes. +- Produces: `ServiceViewSet` becomes retrieve only. `GET /catalog/services/{slug}/` is the + one operation left on it. + +Nothing here has a reader once the component collection and the event feed exist. A column +nothing sorts, filters or renders by does not stay. + +- [ ] **Step 1: Write the failing tests** + +In `api/tests/test_catalog_api.py`, replace the tests that exercised the service list with: + +```python +def test_the_service_list_is_gone(client): + # Discover searches components, and the signed-out board lists + # overall components. Nothing asked a service list a question. + assert resolve_or_none("/catalog/services/") is None + + +def test_the_nested_component_route_is_gone(client): + # `/catalog/components/?service=` serves it, and three other + # screens besides. + service = ServiceFactory() + response = client.get(f"/catalog/services/{service.slug}/components/") + assert response.status_code == 404 + + +def test_the_nested_event_route_is_gone(client): + service = ServiceFactory() + response = client.get(f"/catalog/services/{service.slug}/events/") + assert response.status_code == 404 + + +def test_a_service_still_answers_by_slug(client): + # The service page reads it for the header and the About tab. + service = ServiceFactory(name="Twilio") + response = client.get(reverse("service-detail", args=[service.slug])) + assert response.status_code == 200 + assert response.json()["name"] == "Twilio" + # Nothing renders a description, so it is not in the shape. + assert "description" not in response.json() +``` + +`resolve_or_none` is a helper: + +```python +def resolve_or_none(path): + from django.urls import Resolver404, resolve + + try: + return resolve(path) + except Resolver404: + return None +``` + +In `api/tests/test_board_api.py`: + +```python +def test_the_board_takes_several_severities_at_once(client, board): + # The Severity filter offers all six values. A single exact match + # could not express "everything that needs attention". + client.force_login(board.owner) + response = client.get( + reverse("board-components", args=[board.id]), {"status__severity__in": "0,1,2"} + ) + assert response.status_code == 200 + + +def test_the_event_parameter_is_gone(client, board): + # It named the Home Incidents and Maintenance tabs. Home is Board + # and Updates now, and Updates is `/events/?dashboard=`. + client.force_login(board.owner) + response = client.get( + reverse("board-components", args=[board.id]), {"event": "incident"} + ) + # django-filter ignores an unknown parameter rather than failing, + # so the proof is that it no longer narrows anything. + assert response.json()["aggregates"]["total"] == response.json()["aggregates"]["total"] + assert "by_event_kind" not in response.json()["aggregates"] +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_catalog_api.py tests/test_board_api.py -v` +Expected: FAIL. The list still resolves and `by_event_kind` is still in the envelope. + +- [ ] **Step 3: Reduce the service viewset** + +Replace `ServiceViewSet` in `api/catalog/views.py` with a retrieve-only view: + +```python +class ServiceDetailView(generics.RetrieveAPIView): + """The service page's header and its About tab. + + There is no list. Discover searches components, and the signed-out + board lists overall components, so nothing asked a service + collection a question. + """ + + permission_classes = [AllowAny] + lookup_field = "slug" + serializer_class = ServiceSerializer + filter_backends = [FieldsBackend] + + def get_queryset(self): + return Service.objects.for_display(self.request.user) +``` + +Delete `ServiceAggregateSet`. Its only caller was the list. + +In `api/catalog/urls.py`, drop the router and name the detail route: + +```python +urlpatterns = [ + path("catalog/import/", CatalogImportView.as_view(), name="catalog-import"), + path("catalog/requests/", ServiceRequestView.as_view(), name="catalog-request"), + path("catalog/components/", ComponentListView.as_view(), name="component-list"), + path( + "catalog/components//", + ComponentDetailView.as_view(), + name="component-detail", + ), + path( + "catalog/services//", + ServiceDetailView.as_view(), + name="service-detail", + ), +] +``` + +The component paths sit above the service one, so `components` is never read as a slug. + +- [ ] **Step 4: Delete the columns and the query pieces** + +In `api/catalog/filters.py`, delete `ServiceFilter` and `ServiceEventFilter`. `EventFilter` +in `api/status/filters.py` replaced the second. + +In `api/catalog/queries.py`, delete `WATCHER_COUNT`. `COMPONENT_WATCHER_COUNT` replaced it. + +In `api/catalog/models.py`, delete `Service.is_featured` and `Service.description`. + +In `api/catalog/serializers.py`, remove `"description"` from `ServiceSerializer.Meta.fields`. + +In the adapters, drop `description` from whatever `fetch_service_metadata` returns: + +Run `grep -rn "description" api/polling/` and remove the field from the dataclass and from +each adapter that sets it. Only cState returned one. + +- [ ] **Step 5: Delete the board's event filter and the aggregate** + +In `api/dashboards/filters.py`, delete the `event` filter and its `filter_event` method, +and add the `in` lookup: + +```python +class BoardComponentFilter(filters.FilterSet): + # Declared because a component has a history of statuses and the + # open one is current. There is no `status` relation to generate + # from, so the contract's name points at the annotation. + status__severity = filters.NumberFilter(field_name="severity_now") + status__severity__lte = filters.NumberFilter( + field_name="severity_now", lookup_expr="lte" + ) + status__severity__in = filters.BaseInFilter( + field_name="severity_now", lookup_expr="in" + ) + + class Meta: + model = ServiceComponent + fields = [] +``` + +In `api/common/aggregates.py`, delete `_by_event_kind` and its line in `build`. It counted +the Home Incidents and Maintenance tabs, which no longer exist. + +- [ ] **Step 6: Run the tests** + +Run: +```bash +cd api && uv run python manage.py makemigrations catalog -n drop_service_columns +uv run pytest tests/ -q +``` +Expected: PASS. Fix any test still asserting on `description`, `is_featured` on a service, +`by_event_kind`, or the removed routes. + +- [ ] **Step 7: Update the contract and the diagram** + +In `docs/api/openapi.yaml`: + +- delete the `/catalog/services/` operation +- delete `/catalog/services/{slug}/components/` and `/catalog/services/{slug}/events/` +- delete `description` from the `Service` schema +- delete `by_event_kind` from `StatusAggregates` + +In the v1 spec's ER diagram, delete `description` and `is_featured` from `Service { ... }`. + +- [ ] **Step 8: Prove the contract and the code agree** + +Run: +```bash +cd api && uv run pytest tests/test_contract.py -v +docker compose -f docker-compose.test.yml run --rm docs +``` +Expected: PASS. `test_no_operation_exists_that_the_contract_does_not_document` and +`test_every_documented_operation_exists_in_the_code` are the two that catch a half-done +removal. + +- [ ] **Step 9: Commit** + +```bash +git add api docs +git commit -m "refactor: delete what nothing reads any more + +The service list, both nested routes, ServiceFilter, WATCHER_COUNT, +Service.description and Service.is_featured all served screens the +approved decks replaced. by_event_kind counted tabs that no longer +exist, and the board's event filter named the same two." +``` + +--- + +## Task 13: Admin follows the model + +**Files:** +- Modify: `api/catalog/admin.py` +- Test: `api/tests/test_admin.py` + +**Interfaces:** +- Consumes: `ServiceComponent.is_featured`, `catalog.queries.COMPONENT_WATCHER_COUNT`, + `catalog.models.ServiceRequest` +- Produces: nothing other code reads. + +Featuring a service now means ticking a flag on its overall component. An admin should not +have to know that to do it. + +- [ ] **Step 1: Write the failing tests** + +In `api/tests/test_admin.py`: + +```python +def test_the_service_admin_features_through_the_overall_component(admin_client, db): + # is_featured moved to the component. An admin asks "feature this + # service", so the service page has to answer it. + service = ServiceFactory() + overall = ComponentFactory(service=service, is_overall=True) + response = admin_client.get( + reverse("admin:catalog_service_change", args=[service.pk]) + ) + assert response.status_code == 200 + assert b"is_featured" in response.content + + +def test_the_component_list_shows_watchers(admin_client, db): + # The count is an annotation. A list column that forgets to + # annotate it raises rather than showing a wrong number. + component = ComponentFactory() + track(component) + response = admin_client.get(reverse("admin:catalog_servicecomponent_changelist")) + assert response.status_code == 200 + + +def test_service_requests_are_listed_by_demand(admin_client, db): + # The list is the demand signal for what the catalog is missing, + # so the most-asked-for URL is the first row. + ServiceRequest.objects.create(url="https://a.example", request_count=1) + ServiceRequest.objects.create(url="https://b.example", request_count=9) + response = admin_client.get(reverse("admin:catalog_servicerequest_changelist")) + assert response.status_code == 200 + assert response.content.index(b"b.example") < response.content.index(b"a.example") +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `cd api && uv run pytest tests/test_admin.py -k "features_through or shows_watchers or by_demand" -v` +Expected: FAIL with `NoReverseMatch: 'catalog_servicerequest_changelist' is not a registered +namespace`. + +- [ ] **Step 3: Feature a service from its own page** + +In `api/catalog/admin.py`, add an inline to the service admin: + +```python +class OverallComponentInline(admin.StackedInline): + """Featuring lives on the rollup, and an admin asks about the service. + + `is_featured` is the first key of the suggested sort. Ticking it on + a service's overall component is what surfaces that service, so + that is the question this answers. + """ + + model = ServiceComponent + fields = ["is_featured"] + extra = 0 + max_num = 1 + can_delete = False + verbose_name = "Featuring" + verbose_name_plural = "Featuring" + + def get_queryset(self, request): + return super().get_queryset(request).filter(is_overall=True) + + def has_add_permission(self, request, obj): + # Reconcile makes the rollup. An admin ticks its flag. + return False +``` + +Add `inlines = [OverallComponentInline]` to the service admin, and remove `is_featured` +and `description` from its `list_display`, `list_filter` and `fieldsets`. + +- [ ] **Step 4: Show watchers on the component list** + +In the component admin: + +```python + list_display = [..., "watchers", "is_featured"] + list_filter = [..., "is_featured"] + + def get_queryset(self, request): + # `watcher_count` is an annotation, not a column. A list column + # reading it without this raises rather than showing a wrong + # number. + return super().get_queryset(request).annotate(watcher_count=COMPONENT_WATCHER_COUNT) + + @display(description=_("Watchers"), ordering="watcher_count") + def watchers(self, obj): + return obj.watcher_count +``` + +Import `COMPONENT_WATCHER_COUNT` and drop the `WATCHER_COUNT` import. + +- [ ] **Step 5: Register the requests** + +```python +@admin.register(ServiceRequest) +class ServiceRequestAdmin(ModelAdmin): + """What the catalog is missing, most asked for first.""" + + list_display = ["url", "request_count", "last_requested_at", "created_by"] + ordering = ["-request_count", "-last_requested_at"] + search_fields = ["url"] + readonly_fields = ["url", "request_count", "last_requested_at"] + + def has_add_permission(self, request): + # A row arrives from the app, never from here. + return False +``` + +- [ ] **Step 6: Run the tests** + +Run: `cd api && uv run pytest tests/test_admin.py -v` +Expected: PASS. + +- [ ] **Step 7: Open the pages and read the values back** + +Run: `just dev`, then open each in a browser: + +- `/admin/catalog/service//change/`, the Featuring block shows one checkbox +- `/admin/catalog/servicecomponent/`, the Watchers column shows a number, and sorts +- `/admin/catalog/servicerequest/`, the most-asked URL is the first row + +An edit that ran without error is not an edit that worked. + +- [ ] **Step 8: Run everything and commit** + +Run: `cd api && uv run pytest tests/ -q && uv run ruff check --fix . && uv run ruff format . && python3 ../bin/check_prose.py` +Expected: PASS. + +```bash +git add api +git commit -m "feat: admin features a service from the service page + +is_featured moved to the overall component, and an admin asks to +feature a service rather than a component. The component list shows +watchers, and service requests are ordered by how often they are asked +for." +``` + +--- + +## Task 14: Prove the whole thing in the image CI runs it in + +**Files:** +- Modify: none, unless something fails + +**Interfaces:** +- Consumes: everything above +- Produces: a branch ready for a pull request + +`just check` proves the image rather than the host. What CI proves is then what ships. + +- [ ] **Step 1: Run the full gate** + +Run: `just check` + +Expected: PASS on every step. That is pytest with the coverage gate, ruff, the docs cross-check, +the prose check, and `makemigrations --check`. + +- [ ] **Step 2: Fix whatever it caught, then run it again** + +The likely failures, and what each means: + +| Failure | Cause | +| --- | --- | +| `makemigrations --check` is not clean | a model change with no migration; run `makemigrations` | +| `docs` fails on an unmapped schema | a new schema is missing from `BACKED` or `PLAIN` in `docs/check_docs.py` | +| `docs` fails on a field with no column | the ER diagram is missing a column, or the field belongs in `DERIVED` | +| coverage below 85 | a branch in `system_events.py` with no test, most likely the claim guards | +| prose | a comment over 20 words, or an em dash | + +- [ ] **Step 3: Read the contract once, end to end** + +Run: `just dev`, open `http://.localhost:/`, and read the Scalar page. + +Confirm eighteen operations, and that the three removed ones are absent. The contract is +what the client is generated from, so a wrong description ships to every screen. + +- [ ] **Step 4: Open a pull request** + +```bash +git push -u origin client +gh pr create --fill +``` + +CI runs the same gate on the same image. + +--- + +## Self-Review Notes + +**Spec coverage.** Every section of `docs/specs/2026-09-03-statusboard-client-design.md` +maps to a task, except sections 3, 4, 10, 11 and 12. Those are the client, its build and +its deploy, and they are Plan B. + +| Spec section | Task | +| --- | --- | +| 1, What this supersedes | 1 | +| 5, Removed | 12 | +| 5, Added | 5, 9, 10, 11 | +| 5, Changed | 12 | +| 5, Phase is a declared filter | 9 | +| 5, A tab badge is the collection's total | 9, 10 | +| 5, Discover is one list | 5, 12 | +| 5, Ordering | 4, 5 | +| 6, New model | 11 | +| 6, New columns | 2, 3, 4, 6 | +| 6, Removed columns | 12 | +| 6, Changed | 3, 6, 12 | +| 6, Admin | 13 | +| 6, Choices | 6 | +| 7, A status change is an event | 7, 8 | +| 8, Search | 4 | +| 9, Settings | none needed. The Refresh row was never an endpoint. | + +**Names used across tasks.** `rebuild_ancestry` and `rebuild_search` are defined in Tasks 3 +and 4 and called in both. `descendant_count` replaces `child_count` in Task 3 and is read by +the serializer in the same task. `COMPONENT_WATCHER_COUNT` is defined in Task 2 and read in +Tasks 5 and 13. `EventSource` is defined in Task 6 and read in Tasks 7, 8 and 10. +`SYSTEM_EVENT_MAX_SEVERITY` is defined and used in Task 7, and read again in Task 8. +`claim` is defined in Task 8 and called from `reconcile` in the same task. + +**Ordering constraint.** Task 5 must land before Task 12. The component collection has to +serve the screens before the service list is deleted, or the API is unusable between two +commits on the same branch. From 695dc9194573ff197c2d00e743051bb8d43b8354 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 11:02:41 -0700 Subject: [PATCH 03/94] docs: one account of the API, not two The v1 spec's API, frontend and design sections predate the approved decks. They name tabs that no longer exist and a palette that was replaced. A pointer to the client design replaces them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- docs/specs/2026-08-23-statusboard-design.md | 628 +------------------- 1 file changed, 9 insertions(+), 619 deletions(-) diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 8083415..11b7672 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -666,636 +666,26 @@ Twilio produce one poll every five minutes, not two hundred. A failed fetch sets `unknown` and **never clobbers the last known value**. The UI shows the grey state plus how long it has been stale. -Why the cooldown lives on the service rather than the dashboard or the user is specified in §5 -under **There is no manual refresh**. +**The cooldown stays on the service, globally.** The thing needing protection is somebody +else's status page. A per-user or per-board limit would poll one service twice for two +boards that both track it. A poll also captures the service's **logo** from its own page. A favicon belonging to the parent company rather than the product — Google's mark for Google Slides — is discarded rather than stored: a wrong logo misidentifies the row, while a missing one falls back to an initial and merely looks incomplete. -## 5. API surface +## 5. The API, the client and the design system -**The contract is [`docs/api/openapi.yaml`](../api/openapi.yaml)** — 15 operations, 16 schemas, -with every parameter and response shape. It is the single source of truth; drf-spectacular will -generate it from the code, and it is served through Scalar at the root like your other -projects. This section records only the decisions behind it, so the two cannot drift. +Those three sections are now +[`2026-09-03-statusboard-client-design.md`](2026-09-03-statusboard-client-design.md). -### Identifiers - -`GET /me/` returns your `default_dashboard_id`, and the client goes straight to -`/dashboards/{uuid}/components/`. The name says *default* rather than *the* because v1 having one -per user is a product decision, not a schema one. - -**There is no `GET /dashboards/{uuid}/`.** The board's own fields — `name`, `is_default` — are not -rendered anywhere: the header says "Home", not the dashboard's name. An endpoint returning three -unused fields is a request nobody makes. - -It arrives when a board is shared, because a recipient needs its name and its owner, and the URL -is already the one sharing will use. Adding it then is additive; shipping it now is a guess. - -**UUID is the identity for every model**, from `common.BaseModel`. Path parameters are UUIDs -everywhere except one: `Service` also carries a stable `slug`, used on public catalog routes, -because those URLs get shared during an outage and a UUID would be hostile there. Anything the -user owns stays UUID, so nothing personal is enumerable. Writes always address the UUID. - -An import derives the slug from the host and de-duplicates on collision (`linear`, `linear-2`). - -**`POST /catalog/import/` creates a service from a status page URL.** It detects the provider, -creates the `Service`, its `StatusPage` and its components, and returns the service — `201` when it -created one, `200` when the URL already resolved to one. Nothing is stored about the attempt. - -**`POST /catalog/services/` is deliberately not defined.** Creating a service has several possible -inputs: a URL, a filled-in form, a bulk file. Each takes a different body, so each needs its own -path. If the plain collection `POST` meant "from a URL", the other two would have to invent names -while the standard endpoint sat spent on one special case. - -So none of them takes it. That is a reservation with three known claimants, not a guess about the -future. - -An earlier draft called this `resolve/`, which named the mechanism rather than the thing being -made. - -`Import.service` is the **same shape the catalog list returns**, not the detail shape. One service -representation, whatever produced it — so a client can drop an imported service straight into the -list it already renders. - -That forced the status page fields down into the list shape, because the Add-by-URL screen shows -them in its About block before you press ＋. What was left of `ServiceDetail` — `description` and -`in_catalog_since` — has since folded into `Service` as well, for the reason below. -Renaming keeps the old slug as a redirect rather than breaking shared links. - -### Severity is the state, and there is no status string beside it - -An earlier draft carried both a `severity` integer and a `status` string on every row. Two fields -for one fact can disagree, and one of them is always the redundant copy. Only the integer is -transported; `/meta/` publishes the 0–5 label map once and the client renders from it, so a label -change ships without touching a serializer. - -`status__severity__lte=3` is everything needing attention; `status__severity=4` is maintenance. -There is no named band parameter — a band is just a threshold, and a client can compose -any other one without waiting for a new enum value. `aggregates.by_severity` returns the raw histogram for the same -reason. - -### One list, several screens - -Discover, the suggested lists and the public "down right now" view are the same catalog query with -different parameters. Suggestions are not a separate concept: they are the list with no `q`, sorted -`(-is_featured, severity, -watcher_count, name)`. - -Severity sits ahead of popularity deliberately — a mid-popularity service that is currently broken -is more worth surfacing than a more popular one that is fine, which is the whole premise of the -public view. - -### Components are their own collection - -Cloudflare publishes 109 and nothing caps that, so the service detail reports `component_count` -and the Components tab pages through `/components/`. The tab labels come from the service — -`component_count` and `tracked_component_count` — not from `aggregates`. A -component is a group when `child_count > 0`; there is no separate flag. - -### Filtering, sorting and pagination are server-side, without exception - -There is no cap on board size — one service can contribute 109 components — so the API never -assumes the board fits in one response. `aggregates` covers the whole collection rather than the -page, so a chip is correct on page four. - -There is no client-side fallback. One rule, one implementation. - -### There is no manual refresh - -An earlier draft had `POST /refresh/` behind a header button and a **Refresh now** row menu item. -Both are gone. Reloading the page already shows the freshest data the poller holds, so the button -bought a second path to the same result and carried its own cooldown handling, its own signed-out -routing, and its own failure state. - -**A page load reads stored state. It never triggers a poll.** Every `GET` serves what the last -poll wrote, so the data a reader sees is at most one `interval_seconds` old and the response time -does not depend on a third party being reachable. The alternative — a `GET` that polls anything -stale and waits — would delete the button while moving its cost into every page view, including -the failure mode where someone else's status page is slow. - -**What is kept is the freshness itself.** The header pill still reads "2 min ago" from -`aggregates.oldest_refreshed_at` — the age of the *stalest* row, not an average, so the number is -a floor rather than a flattering middle. Since nothing can be pressed, the pill also answers when -that improves: hovering shows "next check in 3 min" from `aggregates.next_refresh_at`, the soonest -`Service.poller.next_at` in the collection. - -**The cooldown stays on the service, globally.** The thing needing protection is somebody else's -status page, and a per-user or per-board limit would poll one service twice for two boards that -both track it. - -It still has a user-triggered path to guard, just not this one: `POST /catalog/import/` fetches -the URL to detect the provider and read the component list. That is a stranger's request causing -a fetch of a stranger's server, which is the case the cooldown exists for. It also keeps an -import's first poll from stacking on the scheduled one. - -### Sparse fieldsets are a framework concern - -Every endpoint that returns a body accepts `?fields=`. `?fields=id,name,overall_component.status.severity` -returns those three and nothing else. A dotted path prunes inside a nested object rather than -dropping it. - -**This is baked in at the base layer, not written per view.** A `FieldsMixin` on the base -serializer reads the parameter and prunes `self.fields` in `__init__`; a filter backend declares -it so drf-spectacular documents it on every operation without annotation. A view that must not -support it sets `fields_param = None`. - -That matters because the alternative is 13 operations each remembering to add it, and the one that -forgets is the bug. It also keeps the OpenAPI honest: the parameter appears everywhere it works -and nowhere it does not, because both come from the same declaration. - -An unknown field name is a `400`, not a silent empty response. A client asking for a field that no -longer exists should be told. - -### The status page is its own model - -`StatusPage` is a model, `OneToOneField` to `Service`, serialized as `Service.status_page`. It -holds the source and how we read it, separately from what we read: - -```jsonc -"status_page": { - "url": "https://status.twilio.com", - "provider": "statuspage", - "poll_last_success_at": "…", - "poll_next_at": "…", - "poll_interval_seconds": 300, - "poll_cooldown_seconds": 60, - "poll_consecutive_failure_count": 0 -} -``` - -Three loose `status_page_*` fields became one model, and the polling facts that had nowhere -sensible to live joined them. The nesting follows the same rule as everything else here — one -schema, one model — rather than being a serializer that groups fields for looks. -`poll_interval_seconds` was a loose field on the service purely because the About tab shows -"Refreshed every"; it belongs here, next to the backoff that changes it. - -The split is between **the source and the data**. `status_page` is where a service's information -comes from and how reliably we are getting it. `overall_component` is the information. A screen -explaining *why* a status is stale reads the first; a screen showing status reads the second. - -`poll_consecutive_failure_count` and `poll_next_at` make staleness explainable rather than mysterious — the -difference between "Can't check" and "Can't check, we have failed four times and will try again in -twenty minutes". - -`/meta/` keeps deployment defaults for both intervals. A single page can differ from them. - -### A component carries everything happening to it - -Three things happen, and a component carries all three: - -| | on `Component` | how many | -| --- | --- | --- | -| its current state | `status` | — one by definition | -| scheduled work | `upcoming_maintenance` | `upcoming_maintenance_count` | -| a live problem | `active_incident` | `active_incident_count` | - -**A count sits with the projection it counts, over the same set.** A card shows the soonest -unfinished window and the most recent unresolved incident; the counts turn those into "+2 more" -without shipping either list. Both are scoped to what is still live — a count over a wider set -than the item beside it produces "+3 more" pointing at nothing the screen can reach, which is what -happened when `active_incident` was `latest_incident` and returned resolved ones. Both live on -the component, so a board row has them whatever it tracks. - -**The overall component carries the same three, scoped to the whole service.** Its -`upcoming_maintenance` is the soonest unfinished window anywhere in the service, from any -component. Its `active_incident` is the most recent unresolved incident anywhere in it. - -**Tracking the overall component means one card for the service**, instead of one per component. -That is the point of it, and it is why the card compresses: a service with three maintenance -windows still gets one row. - -The row shows `upcoming_maintenance` and, when `upcoming_maintenance_count` is above one, "+2 -more". **Windows are never merged into a combined span** — two windows a week apart are two facts, -and a single merged window would be true of neither. Same for incidents: the row shows -`active_incident` and `active_incident_count` says whether there are others. - -Neither is accompanied by an array. History is unbounded, a row renders from the soonest plus a -count, and the full list is `GET /catalog/services/{slug}/events/` with `kind`. - -That is what makes tracking a service as a whole work. A board row is a component, so a row for -Twilio's overall component must answer "is anything scheduled, is anything broken" for Twilio — -otherwise following the whole service tells you *less* than following one part of it, which is -backwards. - -An earlier draft put the maintenance summary, the incident summary and `active_incident_count` on -`Service` instead. All three are gone: a `Service` nests `overall_component`, and the same values -are read there. One fact, one place, and the board gets them for free because a board row already -is that component. - -Real components are scoped normally — `upcoming_maintenance` is that component's own soonest -unfinished window, and `active_incident` is the most recent unresolved incident naming it through -the `affected_components` M2M. - -**The projection runs one way only.** A component carries its event; an event does not carry its -components. `ServiceEvent.affected_component_ids` used to ship the ids on every row and no screen -read them — the event tabs render titles and update logs, never component names. The M2M filter -`?affected_components=` stays, since django-filter generates it free and narrowing events to one -component is a query a screen may want. The field was a prefetch and bytes for nothing. - -### One envelope, and aggregates that grow without changing it - -Every list returns `{ aggregates, next, results }`. `aggregates` is the one place for anything -computed over the **whole collection** rather than the page. - -An earlier draft had `counts` and `oldest_refreshed_at` sitting side by side at the top level, -which is two aggregations at two different depths. The next figure worth showing would have been a -third key, and a client could not read them generically. - -```jsonc -{ - "aggregates": { "total": 42, "by_severity": {"0": 1, "5": 38}, "oldest_refreshed_at": "…" }, - "next": "cursor…", - "results": [ … ] -} -``` - -`Aggregates` defines the floor — `total`, present everywhere, so it can be read without knowing -which endpoint produced it. An endpoint extends rather than replaces: `StatusAggregates` adds the -severity histogram and `oldest_refreshed_at`; `EventAggregates` adds a histogram by phase. - -**Every chip on a screen must be answerable from one response.** Home has three — All, Incidents, -Maintenance — and the first fetch has to fill all three, or the client makes a request per chip -before it can draw the row of chips. - -`total` gives All. The other two come from `by_event_kind`, which counts the rows each value of -`event` returns. Neither can be read off `by_severity`: a maintenance window three days out leaves -the component at severity 5, and a provider can open an incident on components it still reports as -operational. - -The rule that follows: **if a filter cannot be derived from an existing aggregate, it needs its own -aggregate.** A chip is a filter and a count, and shipping one without the other means the screen -cannot be drawn in one request. - -**An aggregate has to mean the same thing on every endpoint that returns it.** A `tracked` count -was dropped for failing that: on a service's components it duplicated -`Service.tracked_component_count`, and on the board it always equalled `total`, because everything -on a board is tracked by definition. Either a duplicate or a tautology, depending on the caller. A new list endpoint declares its own subclass and the envelope never changes shape. - -In DRF this is one pagination class emitting the three keys, with each viewset supplying -`get_aggregates(queryset)` — so adopting the shape is a method, not a new response schema. - -The rule that makes it worth having: **nothing collection-wide is ever added at the top level.** -A figure that describes the whole collection goes in `aggregates` or it does not ship. - -### One schema, one model - -Serializers are `ModelSerializer`s, so the API shape has to be something a `ModelSerializer` can -actually produce. Every schema therefore maps to a single model, and **related models nest under a -key rather than flattening into the parent**. - -`status` is the case that forced the rule. Severity, `started_at` and `last_refreshed_at` live on -`ComponentStatus`, a separate table from `Component` and further still from `Service`. An earlier -draft merged them into the parent object, which reads fine as JSON but is a hand-assembled -serializer with `source=` on every field and no model behind it. Nested, it is one -`StatusSerializer` over one model, reused on both parents — and a `Service` reaches its status through -the overall component, which is the job that component was introduced for. - -The rule applies transitively, which is easy to miss. A `Service` has no status of its own — its -status *is* its overall component's. An intermediate draft gave `Service` a `status` key with -`source="overall_component.status"`, reaching through a relation to hoist fields onto the parent: the same -mistake one level up. It also carried `overall` as a bare id and `is_tracked` as a third field -describing that same component. All three collapse into one nested object: - -```jsonc -"overall": { "id": "…", "status": { "severity": 1, … }, "is_tracked": true }, -"tracked_component_count": 3 -``` - -One FK, one nested serializer. The response path and the filter path become identical — -`overall_component.status.severity` is read at `?overall_component__status__severity__lte=3`. - -**Tracking is two questions, not one.** Whether the *overall* component is tracked is what the ＋ -on a service row toggles; whether *anything* from the service is tracked decides if it appears on -your board. You can track Twilio SMS without tracking Twilio as a whole. The first is -`overall_component.is_tracked`; the second is `tracked_component_count > 0`, read from the field on a -row rather than filtered on, so no second boolean is carried for something the count already answers -— and the same count renders the "3 tracked" label. - -The count **includes the overall component**, because overall is a component like any other. -Excluding it would reintroduce the service-versus-component split this design exists to remove. - -### One schema per model, and reference schemas where nesting forces them - -There is no list variant and detail variant of the same resource. `ServiceDetail` was `Service` -plus `description` and `in_catalog_since`; `TrackedComponent` was `Component` plus `service`. -Both withheld fields that cost nothing — plain columns, or a `select_related` join — so both -folded into the resource they extended. Trimming a response is `?fields=`, where the client -chooses, rather than a second schema deciding for it. - -`OverallComponent` went the same way. `Service.overall_component` is a plain `Component`, so a -component has one shape everywhere and one renderer draws every row. Six of its fields are fixed -by what an overall component is — `parent` and `path` null, `child_count` 0, `is_overall` true, -`archived_at` null, `name` the service name — and they are returned rather than omitted. A client -that special-cased them would need a second renderer to gain nothing. - -**Reference schemas are a different thing and they stay.** `ServiceRef` and `EventRef` are not -smaller variants of a resource; embedding the full one is impossible or unbounded: - -| Schema | Fields | Why it cannot be the full resource | -| --- | --- | --- | -| `ServiceRef` | id, slug, name, logo | `Component.service`. A `Service` nests `overall_component`, which is a `Component`, which nests its service — a full `Service` there never terminates. `ServiceRef` nests nothing, which is what ends the chain. | -| `EventRef` | id, kind, title, phase, starts_at, ends_at | `active_incident` and `upcoming_maintenance`. A full `ServiceEvent` carries `updates`, a reverse FK with no bound — the same reason the maintenance array was removed. | - -The test is whether the omitted fields are *unavailable* or merely *unwanted*. Unavailable is a -reference schema. Unwanted is `?fields=`. `OverallComponent`'s were neither: they were available -and wanted, just constant. - -`allOf` survives in only one role: a serializer subclassing another **over the same model**. It -never merges two models into one flat object. - -Query parameters use the ORM path, and the nesting is visible in the URL on purpose. - -```python -filterset_fields = {"status__severity": ["exact", "lte"]} # ?status__severity__lte=3 -ordering_fields = ["status__severity", "name", "updated_at"] -``` - -That is django-filter's zero-config behaviour. Renaming them to `severity_lte` buys a shorter URL -and costs a declared `NumberFilter` per field per viewset — a `FilterSet` class to write and keep -in sync every time a field is added. The ORM path also means a client can read a response body and -derive the filter for anything in it, including fields that do not exist yet. - -**The tabs are one axis, and severity is a filter that works inside any of them.** An earlier -draft named them All, Outages and Maintenance. Those straddled two axes: Outages filtered status, -Maintenance filtered events, and All filtered nothing. Every difficulty came from that — the -Maintenance filter had to be hand-written to span a status field and an event relation, and its -count could not come out of `by_severity`. - -Incidents and maintenance are both `ServiceEvent`. Naming the tab after the model puts all three -on the event axis: `event=incident`, `event=maintenance`, and no parameter for All. One generated -filter, one symmetric aggregate. - -Severity then composes with any tab rather than competing with one. -`event=incident&status__severity__lte=3` is written-up incidents that are also down right now. - -`event` is one of the declared filters, and the reason is a Django rule rather than a preference. -`ServiceEvent` and `ServiceComponent` are many-to-many. django-filter applies each query parameter -as its own `.filter()` call, and on a many-to-many every call joins the relation again — so -`?events__kind=incident&events__phase__in=…` lets the two conditions land on two different events. -A component holding one closed incident and one running maintenance satisfies both and matches -neither. The conditions have to sit in a single call: - -```python -def filter_event(self, qs, name, value): - return qs.filter( - events__kind=value, - events__phase__in=ServiceEvent.OPEN_PHASES, - ).distinct() -``` - -`.distinct()` because the join returns a component once per matching event. - -**What each tab can promise.** A provider can drop a component to *Major outage* without writing -an incident, and that row will not appear under Incidents. So the tabs divide as: - -| Tab | Answers | -| --- | --- | -| All | What is wrong — sorted worst-first, whether or not anyone wrote it up | -| Incidents | What has been written up | -| Maintenance | What is planned | - -All is the one that promises completeness, which is why its default ordering is `status__severity` -ascending. Incidents is the narrative, not the alarm. - -Three parameters cannot follow the rule and are declared: -`overall_component__is_tracked` is a per-user annotation with no ORM path behind it, and -`ordering=suggested` is a multi-key sort. Everything else is generated. - -Because these paths are public, **a model field name is an API decision.** It appears in -`?fields=` and in every filter path built from it, so it is chosen to be read, not just stored. - -### What belongs to the user, the device, and the deployment - -Three things were sitting on `/me/` that are not user data, and each moved: - -- **Theme is per device.** Someone runs dark on their phone at night and light on a desktop in a - bright office; syncing it as an account preference makes one of those wrong. It lives in that - browser's local storage and never reaches the server. -- **Poll interval is deployment-wide**, not a preference. A user cannot make us hit somebody - else's status page more often, so it was never theirs to set. -- **The severity label map, page sizes and refresh cooldown** are the same for everyone. - -Those now come from `GET /meta/`, fetched once at app start and cacheable. It also means the 0–5 -labels have exactly one home. What is left on `/me/` is genuinely personal and small: id, email, -`default_dashboard_id`. Nothing on it was patchable once theme left, so `PATCH /me/` is gone — -`GET` and `DELETE` are the whole resource. - -### Public access - -Catalog and incident endpoints are readable without a token; that data is not personal. -`is_tracked` comes back `null` rather than absent, so the client renders `＋` without a second code path. -Adding, refreshing, `/me/` and `/dashboards/` require auth, and public endpoints carry their own -caching and rate limits. - -### Screen-to-endpoint map - -| Screen | Calls | -| --- | --- | -| Home · All | `GET /dashboards/{uuid}/components/` | -| Home · Incidents | `…/components/?event=incident` | -| Home · Maintenance | `…/components/?event=maintenance` | -| Home, signed out | `GET /catalog/services/?overall_component__status__severity__lte=` | -| Header pill | no call — `aggregates.oldest_refreshed_at` and `next_refresh_at`, both already in the list response | -| Row ⋮ → Stop tracking | `DELETE /dashboards/{uuid}/components/{component_id}/` — the only item left in that menu | -| Discover, suggested | `GET /catalog/services/` | -| Discover, typing / no results | `GET /catalog/services/?q=` | -| Add by URL | `POST /catalog/import/` | -| Service · Components + filter | `GET /catalog/services/{slug}/components/?is_tracked=` | -| Service · Incidents | `GET /catalog/services/{slug}/events/?kind=incident` — both groups; the tab renders ACTIVE and RESOLVED, so it must not filter on `ends_at__isnull` | -| Service · Maintenance | `GET /catalog/services/{slug}/events/?kind=maintenance` | -| Service · About | `GET /catalog/services/{slug}/` — Website, Status page, Provider, Components, Active incidents, In catalog since, Refreshed every | -| Service · Add all / Remove all | the same `POST` / `DELETE`, repeated per component | -| ＋ on any row | `POST /dashboards/{uuid}/components/` | -| Sign in | `magic-link` → `verify` | -| App start | `GET /meta/` | -| Settings | `GET /me/`, `GET /meta/`, `DELETE /me/`; theme is device-local | - -### The row is one component, used everywhere - -Home, Discover and the service screen render the same card. Line one is the mark and the name; -line two is the status and its duration; line three is context. Nothing shares a line, because a -232px phone row cannot hold two things that must not truncate. - -**Line three is the only variable, and what it holds follows from what the row is.** - -| the row is | line three | -| --- | --- | -| a service you track nothing of | `7 components` — what is on offer | -| a service you track part of | `Tracking [overall] [3 of 42]` | -| a component on your board | its parent breadcrumb, `Twilio › Programmable Messaging` | - -A board row never shows a tracking count. The count answers *how much of this service am I -watching*, which is a question about a **service**; a board row is already one tracked component, -so the answer would always be "this one". - -Tracking state renders as **one token per subscription** rather than a sentence — `overall` for the -summary row, `3 of 42` for components, both when you take both. That maps one-to-one onto -`overall_component.is_tracked` and `tracked_component_count`, so the client assembles no strings, and "all" -is simply `118 of 118` with no extra vocabulary. - -**The duration is always shown**, including on operational rows, where it reads as time since the -service last left operational — the same `status.started_at` every other state uses. Maintenance is -the one case that would state its timing twice, so its window renders in that slot rather than -below, and always relatively: `in 4 hrs` before it starts, `ends in 40 min` while it runs. - -**The overall component is titled by where you are standing.** On a service screen the header -already says Twilio, so the row is `Overall status` with `All services` beneath. On your board -nothing else names the service, so it is `Twilio` with `Overall status` beneath. Same component, -same `is_overall` flag, different useful title. - -**Marks appear only where rows come from different services** — Home and Discover. On a service -screen every row belongs to the same service, so forty-two identical marks are noise and the slot -is dropped. Where a service has no usable logo the row falls back to its initial in the same -circle, so a list with patchy coverage keeps one left edge. - -## 6. Frontend - -React + TS + Vite + Tailwind + TanStack Query + `vite-plugin-pwa`. - -### Navigation - -Three destinations: **Home**, **Discover**, and a third slot that is **Sign in** when logged out -and **Settings** when logged in. Signed out and signed in are the same screens; only the buttons -differ. - -- Mobile: bottom pill, inset to the content width, fully rounded -- Desktop: top bar; Sign in sits right as a primary button - -### Screens - -| Screen | States | -| --- | --- | -| Home | All · Incidents · Maintenance; empty, unreachable, row menu | -| Discover | suggested, typing, no results | -| Add by URL | detected, not found | -| Service | Components · Incidents · Maintenance · About; not tracked, outage, unknown | -| Sign in | request, check your email, verify | -| Settings | preferences, account, install | - -### Chips and filters - -`All` · `Incidents` · `Maintenance`. The two named tabs filter on event kind; All applies no -filter and sorts worst-first. Counts describe whatever the tab shows: the catalog when signed out, -your tracked set when signed in. Signed out shows no counts, because nothing is yours. - -**Colour follows the dot, and the dot is always the true severity.** Maintenance already under way -is severity 4, so its row is blue and reads "Maintenance ends in 2 hrs". Maintenance three days out -leaves the component operational, so its row is **green** and reads "Maintenance in 3 days". The -tab decides which fact the line describes; severity decides what colour it is. A row never shows a -colour the component is not in. - -Sort is **Smart** by default (`ordering=suggested`): tracked first, then severity ascending -(worst first), then name. - -### No offline mode - -Statusboard reports on remote services. With no network there is nothing true to report — a cached -board would show yesterday's answer to today's question, which is worse than saying nothing. - -So there is no stale-cache mode, no offline banner and no dimmed rows. A failed request shows a -plain "can't reach statusboard" state with a retry. The service worker exists only to make the app -installable; it does not cache API responses. - -The `unknown` status is a different thing and stays: that is *our* server failing to reach a -provider, which we can report accurately because we are online and they are not. - -### Install - -No prompt on first visit. A dismissible banner on the third visit, plus a row in Settings. -**iOS has no programmatic install prompt**, so Safari gets "tap Share, then Add to Home Screen" -while Android and desktop Chrome get a real button. - ---- - -## 7. Design system - -### Vocabulary - -**You track. The app refreshes. The catalog gets no verb.** - -- `＋` starts tracking; `⋮` opens the menu that stops it. No tick, no lock icon. -- Signed out, `＋` is still `＋` and opens Sign in. -- "Refreshed every 5 min", never "updated" — the Incidents tab already uses *updates* for - provider posts — and never "monitored". -- "In catalog since", never "tracking since". - -### Row anatomy - -Identical on both platforms; desktop only changes the container. - -``` -[logo] Twilio ● - Major outage · 38 min - 42 components ⋮ -``` - -- **Logo** top-left, neutral disc with an initial. Only where rows come from different services — - on a service screen every row is the same service, so the slot is dropped. No provider publishes - per-component icons. -- **Status** in words, in its severity colour, with duration in mono beside it -- **Meta** last: `42 components` for a service, the breadcrumb for a component, - `Overall status` for the rollup row. **On the Maintenance tab every row shows its window - instead**, in local time. -- **Dot** 15px, top-right. **Action** bottom-right. - -The subtitle says what the row *is*, never what you have done with it. - -### Colour - -| Token | Value | -| --- | --- | -| Ground | `#F5EDD6` | -| Card | `#FFFFFF` | -| Nav / ink | `#29140A` / `#0D0D0D` | -| Accent | `#32CDB3` (notifications toggle only) | - -Status dots use the palette above at **80% fill opacity**, as Figma sets them. - -**Status text uses darkened variants** — `#1C7A3E`, `#A35F00`, `#B3180B`, `#0B6EA0` — because -`#00E54D` on white is about 1.8:1 and unreadable at 11px. Dots keep the exact Figma colours. - -Rules: - -- Status colour is visible wherever services are listed, including the catalog -- Green means operational and nothing else — buttons use brand dark -- Count pills are grey: a count is a quantity, not a severity -- Secondary buttons are transparent with a `#D6CCB8` border; primary is dark fill - -### The mark - -The statusboard logo carries the status dot wherever it appears — brand row, tab bar, favicon, -app icon. **One dot, the lower-left counter**, coloured by the worst status you track. It is the -position your artwork already varies (red and grey); the upper-right is green in every variant. -Scheduled maintenance keeps it green — planned work does not need you. - -Signed out, nothing is tracked, so the counter stays hollow. - -### Icons - -Home and Discover use the Figma marks. Settings and Sign in use Lucide (`settings`, `log-in`), -and the back arrow is Lucide `chevron-left`. The Figma tab-bar exports embed their labels as -vector paths — extract the glyph only. - -Open: the Figma marks are filled and Lucide is stroked, so the nav mixes two weights. Either draw -filled versions of those two or move Discover to Lucide, leaving Home as the only filled mark. - -### Desktop differences - -Only three, each forced by width: - -1. Top nav instead of a bottom pill -2. Rows flow into `repeat(auto-fill, minmax(212px, 1fr))` -3. A field that filters a list spans the list; a form that *is* the task gets a column — - 360px form card, 560px for Add-by-URL - -Plus one intentional divergence: the service title keeps its status label, which mobile drops for -width. +They described a palette, tabs and a row that the approved decks replaced on 2026-08-28. +Keeping a second account of one API is how two answers to the same question drift apart. --- -## 8. Testing, infra, deploy +## 6. Testing, infra, deploy ### Testing From 0c62ffc1d8f1527f27b896b33f1de185a540d573 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 11:17:46 -0700 Subject: [PATCH 04/94] feat: a component is featured and counts its own watchers Discover searches components, so the suggested sort needs both keys on a component. The watcher count is an annotation, not a column: the service had one, and four write paths never reached the signal that kept it true. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- .../migrations/0003_component_is_featured.py | 22 +++++++++++++ api/catalog/models.py | 4 +++ api/catalog/queries.py | 5 +++ api/dashboards/views.py | 1 - api/tests/factories.py | 18 +++++++++-- api/tests/test_board_api.py | 31 +++++++++---------- api/tests/test_catalog_models.py | 24 +++++++++++++- docs/specs/2026-08-23-statusboard-design.md | 1 + 8 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 api/catalog/migrations/0003_component_is_featured.py diff --git a/api/catalog/migrations/0003_component_is_featured.py b/api/catalog/migrations/0003_component_is_featured.py new file mode 100644 index 0000000..7521347 --- /dev/null +++ b/api/catalog/migrations/0003_component_is_featured.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.17 on 2026-09-03 18:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0002_drop_watcher_count"), + ] + + operations = [ + migrations.AddField( + model_name="historicalservicecomponent", + name="is_featured", + field=models.BooleanField(default=False, verbose_name="Featured"), + ), + migrations.AddField( + model_name="servicecomponent", + name="is_featured", + field=models.BooleanField(default=False, verbose_name="Featured"), + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index c467fd6..e5a4b32 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -266,6 +266,10 @@ class ServiceComponent(BaseModel): status_page_order = models.IntegerField(verbose_name="Page order", default=0) is_overall = models.BooleanField(verbose_name="Overall", default=False) + # Ticked on the overall component to feature a whole service. It is + # the first key of the suggested sort. On day one that is the whole + # sort: every watcher count starts at zero. + is_featured = models.BooleanField(verbose_name="Featured", default=False) # Archiving is the flag. The date is kept because `updated_at` # moves on every save, so a later rename would erase it. `save` sets # the date from the flag, and a constraint holds them together. diff --git a/api/catalog/queries.py b/api/catalog/queries.py index 56a8bd4..26df89b 100644 --- a/api/catalog/queries.py +++ b/api/catalog/queries.py @@ -29,6 +29,11 @@ # only it pays for a distinct count over three joins. WATCHER_COUNT = Count("components__boards__owner", distinct=True) +# Who tracks one component. `DashboardItem` points straight at it, so +# this is one join. The service version needed three, and only the +# service sort read it. +COMPONENT_WATCHER_COUNT = Count("boards__owner", distinct=True) + def is_tracked(field="pk"): """Whether anybody tracks any part of a service. diff --git a/api/dashboards/views.py b/api/dashboards/views.py index 8275d2a..4c5ec93 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -63,7 +63,6 @@ def create(self, request, *args, **kwargs): component = get_object_or_404( ServiceComponent, id=request.data.get("component_id") ) - # DashboardItem.save keeps the service's watcher count true. _, created = DashboardItem.objects.get_or_create( dashboard=board, component=component, diff --git a/api/tests/factories.py b/api/tests/factories.py index 41ccee5..8356261 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -1,9 +1,17 @@ import factory +from django.contrib.auth import get_user_model from catalog.models import Service, ServiceComponent, StatusPage from polling.models import Poller +class UserFactory(factory.django.DjangoModelFactory): + class Meta: + model = get_user_model() + + email = factory.Sequence(lambda n: f"watcher{n}@example.test") + + class ServiceFactory(factory.django.DjangoModelFactory): class Meta: model = Service @@ -79,8 +87,12 @@ def track(component, user=None): return DashboardItem.objects.create(dashboard=board, component=component) -def watchers(service): +def watchers(component): """How many people track it, counted the way the app counts.""" - from catalog.queries import WATCHER_COUNT + from catalog.queries import COMPONENT_WATCHER_COUNT - return Service.objects.annotate(n=WATCHER_COUNT).get(pk=service.pk).n + return ( + ServiceComponent.objects.annotate(n=COMPONENT_WATCHER_COUNT) + .get(pk=component.pk) + .n + ) diff --git a/api/tests/test_board_api.py b/api/tests/test_board_api.py index 14e3866..8b3c9a7 100644 --- a/api/tests/test_board_api.py +++ b/api/tests/test_board_api.py @@ -179,7 +179,7 @@ def test_tracking_a_component_adds_it_and_bumps_the_watcher_count(client, board) response = client.post(url, {"component_id": str(component.id)}, format="json") assert response.status_code == 201 assert DashboardItem.objects.filter(dashboard=board, component=component).exists() - assert watchers(component.service) == 1 + assert watchers(component) == 1 @pytest.mark.django_db @@ -212,19 +212,19 @@ def test_a_board_belonging_to_someone_else_is_not_readable(client): @pytest.mark.django_db def test_watcher_count_is_distinct_users_not_tracked_items(client, board): - # Someone tracking five Twilio components is one watcher. Counting - # items would let one person outrank a crowd in the suggestion order. - service = ServiceFactory() - StatusPageFactory(service=service) - url = reverse("board-components", kwargs={"uuid": board.id}) - for external_id in ("a", "b", "c"): - component = ComponentFactory(service=service, external_id=external_id) + # One person tracking a component from two boards is one watcher. + # Counting items would let one person outrank a crowd in the + # suggestion order. + component = ComponentFactory() + second_board = Dashboard.objects.create(owner=board.owner, name="Second") + for target in (board, second_board): + url = reverse("board-components", kwargs={"uuid": target.id}) client.post(url, {"component_id": str(component.id)}, format="json") - assert watchers(service) == 1 + assert watchers(component) == 1 @pytest.mark.django_db -def test_two_people_tracking_one_service_count_twice(client, board): +def test_two_people_tracking_one_component_count_twice(client, board): service = ServiceFactory() StatusPageFactory(service=service) component = ComponentFactory(service=service) @@ -233,21 +233,20 @@ def test_two_people_tracking_one_service_count_twice(client, board): other = Dashboard.objects.get(owner=User.objects.create(email="second@b.com")) DashboardItem.objects.create(dashboard=other, component=component) - assert watchers(service) == 2 + assert watchers(component) == 2 @pytest.mark.django_db def test_untracking_the_last_component_drops_the_watcher(client, board): component = _track(board) - service = component.service - assert watchers(service) == 1 + assert watchers(component) == 1 client.delete( reverse( "board-component-detail", kwargs={"uuid": board.id, "component_id": component.id}, ) ) - assert watchers(service) == 0 + assert watchers(component) == 0 @pytest.mark.django_db @@ -271,7 +270,7 @@ def test_the_watcher_count_is_right_however_a_row_arrives_or_goes(how): user = get_user_model().objects.create(email="watcher@b.com") board = Dashboard.objects.create(owner=user, name="B") DashboardItem.objects.create(dashboard=board, component=component) - assert watchers(service) == 1 + assert watchers(component) == 1 if how == "the admin": DashboardItem.objects.get(dashboard=board, component=component).delete() @@ -283,4 +282,4 @@ def test_the_watcher_count_is_right_however_a_row_arrives_or_goes(how): else: DashboardItem.objects.all().delete() - assert watchers(service) == 0 + assert watchers(component) == 0 diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index de4fb41..68715ad 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -2,7 +2,29 @@ from django.db.utils import IntegrityError from catalog.models import Service, ServiceComponent, StatusPage -from tests.factories import ComponentFactory, PollerFactory, ServiceFactory +from tests.factories import ( + ComponentFactory, + PollerFactory, + ServiceFactory, + UserFactory, + track, + watchers, +) + + +def test_a_components_watcher_count_is_distinct_users(db): + # Two boards holding one component is two watchers. One person + # holding it twice is impossible, and one holding a sibling too is + # still one watcher of this component. + component = ComponentFactory() + sibling = ComponentFactory(service=component.service) + first = UserFactory() + track(component, user=first) + track(sibling, user=first) + track(component, user=UserFactory()) + + assert watchers(component) == 2 + assert watchers(sibling) == 1 @pytest.mark.django_db diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 11b7672..4c42f70 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -159,6 +159,7 @@ erDiagram uuid parent_id FK "self, null at top level" int status_page_order bool is_overall + bool is_featured "first key of the suggested sort" datetime archived_at "set when it stops being published" } ComponentStatus { From ff818ef46306b9dc993700196c2edd98ab4c8bc3 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 11:29:28 -0700 Subject: [PATCH 05/94] feat: a component stores the chain above it A component's Components tab lists every descendant. A self-FK lookup returns one level, and walking the chain per row cannot use an index. Reconcile writes the array, and a reparent rewrites the subtree. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- .../migrations/0004_component_ancestor_ids.py | 70 +++++++++++++++++++ api/catalog/models.py | 53 ++++++++++++-- api/catalog/serializers.py | 8 +-- api/polling/reconcile.py | 34 +++++++++ api/tests/test_catalog_api.py | 2 +- api/tests/test_catalog_models.py | 36 ++++++++++ api/tests/test_reconcile.py | 63 ++++++++++++++++- docs/api/openapi.yaml | 7 +- docs/check_docs.py | 2 +- docs/specs/2026-08-23-statusboard-design.md | 3 +- 10 files changed, 262 insertions(+), 16 deletions(-) create mode 100644 api/catalog/migrations/0004_component_ancestor_ids.py diff --git a/api/catalog/migrations/0004_component_ancestor_ids.py b/api/catalog/migrations/0004_component_ancestor_ids.py new file mode 100644 index 0000000..a1537a3 --- /dev/null +++ b/api/catalog/migrations/0004_component_ancestor_ids.py @@ -0,0 +1,70 @@ +# Generated by Django 5.2.17 on 2026-09-03 18:25 + +import django.contrib.postgres.fields +import django.contrib.postgres.indexes +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0003_component_is_featured"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="historicalservicecomponent", + name="ancestor_ids", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), + blank=True, + default=list, + editable=False, + size=None, + ), + ), + migrations.AddField( + model_name="servicecomponent", + name="ancestor_ids", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.UUIDField(), + blank=True, + default=list, + editable=False, + size=None, + ), + ), + # The column cannot start empty. `polling.reconcile` fills it, + # and it runs only for a poller in `PollerQuerySet.active()`. A + # service nobody tracks is never polled again, so its rows would + # read as having no ancestor for good. + # + # A cycle terminates the walk on its own. Every row in one has a + # parent, so no cycle is reachable from the roots the base term + # selects. Those rows keep the empty default. + migrations.RunSQL( + sql=""" + WITH RECURSIVE tree AS ( + SELECT id, ARRAY[]::uuid[] AS ancestors + FROM catalog_servicecomponent + WHERE parent_id IS NULL + UNION ALL + SELECT child.id, tree.ancestors || tree.id + FROM catalog_servicecomponent child + JOIN tree ON child.parent_id = tree.id + ) + UPDATE catalog_servicecomponent AS row + SET ancestor_ids = tree.ancestors + FROM tree + WHERE tree.id = row.id; + """, + reverse_sql=migrations.RunSQL.noop, + ), + migrations.AddIndex( + model_name="servicecomponent", + index=django.contrib.postgres.indexes.GinIndex( + fields=["ancestor_ids"], name="components_by_ancestor" + ), + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index e5a4b32..9c24298 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -1,7 +1,10 @@ from urllib.parse import urlparse, urlunparse +from django.contrib.postgres.fields import ArrayField +from django.contrib.postgres.indexes import GinIndex from django.core.exceptions import ValidationError from django.db import models +from django.db.models.functions import Coalesce from django.utils import timezone from django.utils.text import slugify from simple_history.models import HistoricalRecords @@ -200,6 +203,32 @@ def normalise_url(url: str) -> str: ) +def _descendant_count(): + """How many components sit under each row, in one query for the page. + + `related_count` cannot express this. It compares a column to the + outer key. `ancestor_ids` holds an array, so the outer key goes in + as a one element array of its own. + + The constant in `values` is what makes this one aggregate. Without + it Django counts per row and returns one row a component, which a + subquery in a SELECT list refuses. + """ + key = models.Func( + models.OuterRef("pk"), + template="ARRAY[%(expressions)s]", + output_field=ArrayField(models.UUIDField()), + ) + counted = ( + ServiceComponent.objects.filter(ancestor_ids__contains=key) + .order_by() + .values(one_group=models.Value(1)) + .annotate(total=models.Count("pk")) + .values("total") + ) + return Coalesce(models.Subquery(counted, output_field=models.IntegerField()), 0) + + class ServiceComponentQuerySet(models.QuerySet): def for_display(self, user=None): """Everything `ComponentSerializer` reads, in four queries. @@ -227,7 +256,7 @@ def for_display(self, user=None): return ( self.select_related("service", "parent", "parent__parent") .annotate( - _child_count=related_count(self.model.objects, "parent"), + _descendant_count=_descendant_count(), _is_tracked=tracked, ) .prefetch_related( @@ -262,6 +291,13 @@ class ServiceComponent(BaseModel): on_delete=models.SET_NULL, related_name="children", ) + # The chain above this one, root first, written by reconcile. A + # component's Components tab lists every descendant, and a self-FK + # lookup returns one level. Walking the chain per row cannot use an + # index either. + ancestor_ids = ArrayField( + models.UUIDField(), default=list, blank=True, editable=False + ) objects = ServiceComponentQuerySet.as_manager() status_page_order = models.IntegerField(verbose_name="Page order", default=0) @@ -302,6 +338,9 @@ class Meta(BaseModel.Meta): name="archived_flag_and_date_agree", ), ] + indexes = [ + GinIndex(fields=["ancestor_ids"], name="components_by_ancestor"), + ] def live_events(self, kind): """The live events of one kind. @@ -321,12 +360,16 @@ def open_status(self): return self.statuses.filter(ended_at__isnull=True).first() return prepared[0] if prepared else None - def child_count(self): - """How many components sit under this one.""" + def descendant_count(self): + """How many components sit anywhere under this one.""" if self.is_overall: return 0 - prepared = getattr(self, "_child_count", None) - return self.children.count() if prepared is None else prepared + prepared = getattr(self, "_descendant_count", None) + if prepared is None: + return ServiceComponent.objects.filter( + ancestor_ids__contains=[self.pk] + ).count() + return prepared def is_tracked_by(self, user): """Whether this user has it on a board. Null if nobody asked.""" diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index 402c265..66360c3 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -54,7 +54,7 @@ class Meta: class ComponentSerializer(ViewerMixin, FieldsMixin, serializers.ModelSerializer): status = serializers.SerializerMethodField() path = serializers.SerializerMethodField() - child_count = serializers.SerializerMethodField() + descendant_count = serializers.SerializerMethodField() upcoming_maintenance = serializers.SerializerMethodField() upcoming_maintenance_count = serializers.SerializerMethodField() active_incident = serializers.SerializerMethodField() @@ -69,7 +69,7 @@ class Meta: "name", "path", "parent", - "child_count", + "descendant_count", "is_overall", "archived_at", "status", @@ -108,8 +108,8 @@ def get_path(self, row): return PathNodeSerializer(row.ancestors, many=True).data @extend_schema_field(serializers.IntegerField()) - def get_child_count(self, row): - return row.child_count() + def get_descendant_count(self, row): + return row.descendant_count() @extend_schema_field(EventRefSerializer(allow_null=True)) def get_upcoming_maintenance(self, row): diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index 604c665..4c32f90 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -77,9 +77,43 @@ def _upsert_components(service, components, author): row.parent = parent row.updated_by = author row.save(update_fields=["parent", "updated_by"]) + rebuild_ancestry(service) return rows +def rebuild_ancestry(service): + """Write every component's chain of ancestors, root first. + + A rename does not touch this. A reparent moves a whole subtree, so + the pass rewrites the service rather than one row. A service holds + hundreds of components, not millions. + """ + rows = list(ServiceComponent.objects.filter(service=service)) + # The walk reads this map, not the database. A deep tree costs no + # extra query. + parents = {row.id: row.parent_id for row in rows} + + def chain(node): + # The column points at its own table, so the data allows a loop. + # `seen` ends the walk. Without it bad data hangs the worker. + walked, seen, up = [], {node}, parents.get(node) + while up is not None and up not in seen: + seen.add(up) + walked.append(up) + up = parents.get(up) + return list(reversed(walked)) + + # Only the rows that moved. Most polls change no parent, and this + # runs on every poll of every service. + changed = [] + for row in rows: + computed = chain(row.id) + if row.ancestor_ids != computed: + row.ancestor_ids = computed + changed.append(row) + ServiceComponent.objects.bulk_update(changed, ["ancestor_ids"]) + + def _archive_vanished(service, components, author): seen = {c.external_id for c in components} # A bulk update never reaches `save`, so both columns are written diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index 88224bf..153ec03 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -67,7 +67,7 @@ def test_the_overall_component_is_a_plain_component(): "name", "path", "parent", - "child_count", + "descendant_count", "is_overall", "archived_at", "status", diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 68715ad..523c18a 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -2,6 +2,7 @@ from django.db.utils import IntegrityError from catalog.models import Service, ServiceComponent, StatusPage +from polling.reconcile import rebuild_ancestry from tests.factories import ( ComponentFactory, PollerFactory, @@ -156,3 +157,38 @@ def test_the_flag_and_the_date_cannot_disagree(): component = ComponentFactory() with pytest.raises(IntegrityError), transaction.atomic(): ServiceComponent.objects.filter(pk=component.pk).update(is_archived=True) + + +def test_ancestor_ids_are_stored_top_down(db): + # A descendant query reads this array. Walking `parent` per row + # cannot use an index, and `for_display` only selects two levels up. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + + leaf.refresh_from_db() + assert leaf.ancestor_ids == [top.id, middle.id] + + # Every descendant, not the direct children. `top` has two below it. + assert ServiceComponent.objects.filter(ancestor_ids__contains=[top.id]).count() == 2 + + +def test_for_display_counts_descendants_without_a_query_a_row( + db, django_assert_num_queries +): + # The count is why `for_display` exists. Asked per row, a page of + # fifty components costs fifty extra queries. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + + rows = list(ServiceComponent.objects.for_display()) + with django_assert_num_queries(0): + counts = sorted(row.descendant_count() for row in rows) + + # Two below the top, one below the middle, none below the leaf. + assert counts == [0, 1, 2] diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index 7c7f78f..822d17a 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -7,7 +7,7 @@ NormalisedEvent, NormalisedUpdate, ) -from polling.reconcile import apply_fetch +from polling.reconcile import apply_fetch, rebuild_ancestry from status.choices import EventKind, IncidentPhase, Severity, StatusSource from status.models import ComponentStatus, ServiceEvent from tests.factories import ComponentFactory, ServiceFactory @@ -243,3 +243,64 @@ def test_a_second_poll_does_not_rewrite_who_made_the_row(): again = ServiceComponent.objects.get() assert again.created_by_id == first.created_by_id assert again.created_at == first.created_at + + +@pytest.mark.django_db +def test_a_poll_writes_the_ancestor_chain(): + # Nothing else writes the column. Without this pass a component's + # Components tab lists no descendant, whatever the provider nests. + service = ServiceFactory() + apply_fetch( + service, + [_component(external_id="a"), _component(external_id="b", parent="a")], + [], + StatusSource.PROVIDER, + ) + + top = ServiceComponent.objects.get(external_id="a") + child = ServiceComponent.objects.get(external_id="b") + assert top.ancestor_ids == [] + assert child.ancestor_ids == [top.id] + + +@pytest.mark.django_db +def test_a_reparent_rewrites_the_subtree(): + # The chain of a grandchild changes when its parent moves, and + # nothing tells the grandchild. The pass rewrites the service. + service = ServiceFactory() + nested = [ + _component(external_id="a"), + _component(external_id="b", parent="a"), + _component(external_id="c", parent="b"), + ] + apply_fetch(service, nested, [], StatusSource.PROVIDER) + + moved = [ + _component(external_id="a"), + _component(external_id="b"), + _component(external_id="c", parent="b"), + ] + apply_fetch(service, moved, [], StatusSource.PROVIDER) + + assert ServiceComponent.objects.get(external_id="c").ancestor_ids == [ + ServiceComponent.objects.get(external_id="b").id + ] + + +@pytest.mark.django_db +def test_a_loop_in_the_parent_column_does_not_hang_the_pass(): + # The column points at its own table, so bad data can make a cycle. + # Without the guard the walk never reaches a root, and the worker + # never finishes the poll that called it. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service, parent=first) + # `update` skips `clean`, which is the only thing refusing this. + ServiceComponent.objects.filter(pk=first.pk).update(parent=second) + + rebuild_ancestry(service) + + first.refresh_from_db() + second.refresh_from_db() + assert first.ancestor_ids == [second.id] + assert second.ancestor_ids == [first.id] diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index da3ec24..7dc661d 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -722,7 +722,7 @@ components: It is a plain `Component`, the same shape a component has anywhere else, so one renderer draws every row. Six of its fields are fixed by what an overall component is: - `parent` and `path` are null, `child_count` is 0, `is_overall` is true, `archived_at` + `parent` and `path` are null, `descendant_count` is 0, `is_overall` is true, `archived_at` is null, and `name` is the service name. They are returned rather than omitted — a client that special-cased them would need a second renderer to gain nothing. @@ -774,10 +774,11 @@ components: nothing. items: { $ref: '#/components/schemas/PathNode' } parent: { type: string, format: uuid, nullable: true } - child_count: + descendant_count: type: integer description: | - A component is a group when this is above zero. There is no is_group flag. + How many components sit anywhere under this one. A component is a group when this + is above zero. There is no is_group flag. Always 0 on the overall component. It is a peer of the top-level components, not their parent. diff --git a/docs/check_docs.py b/docs/check_docs.py index d77920e..a3590e2 100755 --- a/docs/check_docs.py +++ b/docs/check_docs.py @@ -60,7 +60,7 @@ "Poller.cooldown_seconds": "own override or /meta/ default", "Service.poller": "OneToOneField", "Component.path": "walk of parent", - "Component.child_count": "Count of reverse parent", + "Component.descendant_count": "Count of rows whose ancestor_ids holds this id", "Component.is_tracked": "per-user annotation over DashboardItem", "Component.upcoming_maintenance": "soonest unfinished maintenance event across that M2M", "Component.upcoming_maintenance_count": "Count of unfinished maintenance across that M2M", diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 4c42f70..268add1 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -157,6 +157,7 @@ erDiagram string external_id "unique with service" string name uuid parent_id FK "self, null at top level" + uuid[] ancestor_ids "root first, written by reconcile" int status_page_order bool is_overall bool is_featured "first key of the suggested sort" @@ -397,7 +398,7 @@ Unique on `(dashboard, component)`. ### Every service has an overall component Statusboard creates one component per service, `is_overall = true`, `parent = null`, named -"All services". `path` is null, since it has no ancestry, and `child_count` is 0 — it is a peer of +"All services". `path` is null, since it has no ancestry, and `descendant_count` is 0 — it is a peer of the top-level components rather than their parent. **`component_count` excludes it.** The number equals what the provider's own status page claims, From 13b5238642331eeddcbdd44f74d20dafd2d2c6e1 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 11:44:17 -0700 Subject: [PATCH 06/94] docs: plans sit beside specs, not under a tool's name `docs/superpowers/plans/` named the tool that generated the files, which tells a reader nothing and buries them a level deeper than the specs. `docs/` now holds three peers: api, plans and specs. The markdownlint exclude named the old path, so the move would have put five thousand lines of unreflowed prose in front of the hook. The backend plan ended with a handoff section describing how to execute a plan that is already executed. It went with the move. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- .pre-commit-config.yaml | 7 +++---- .../plans/2026-08-26-statusboard-backend.md | 15 ++------------- .../plans/2026-09-03-statusboard-api.md | 0 3 files changed, 5 insertions(+), 17 deletions(-) rename docs/{superpowers => }/plans/2026-08-26-statusboard-backend.md (99%) rename docs/{superpowers => }/plans/2026-09-03-statusboard-api.md (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d3fb09..d9c9452 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,10 +9,9 @@ repos: rev: v0.42.0 hooks: - id: markdownlint - # Two approved documents have never passed this hook: the design - # spec and the plan it was built from. Reflowing either would be - # a huge diff on prose nobody asked to rewrite. - exclude: ^docs/(specs|superpowers)/ + # Specs and plans have never passed this hook. Reflowing prose + # nobody asked to rewrite would be a huge diff. + exclude: ^docs/(specs|plans)/ - repo: https://github.com/astral-sh/ruff-pre-commit # Must match the ruff in api/uv.lock. On different versions the hook # and `just lint` format differently and undo each other, so a commit diff --git a/docs/superpowers/plans/2026-08-26-statusboard-backend.md b/docs/plans/2026-08-26-statusboard-backend.md similarity index 99% rename from docs/superpowers/plans/2026-08-26-statusboard-backend.md rename to docs/plans/2026-08-26-statusboard-backend.md index 3ca2ffc..3f9271f 100644 --- a/docs/superpowers/plans/2026-08-26-statusboard-backend.md +++ b/docs/plans/2026-08-26-statusboard-backend.md @@ -8,8 +8,8 @@ **Tech Stack:** Django 5, DRF, SimpleJWT, django-filter, drf-spectacular, django-unfold, django-celery-beat, Celery + Redis, Postgres 17, uv, ruff, pytest + pytest-django + factory_boy. -**Spec:** [`docs/specs/2026-08-23-statusboard-design.md`](../../specs/2026-08-23-statusboard-design.md) -**Contract:** [`docs/api/openapi.yaml`](../../api/openapi.yaml) +**Spec:** [`docs/specs/2026-08-23-statusboard-design.md`](../specs/2026-08-23-statusboard-design.md) +**Contract:** [`docs/api/openapi.yaml`](../api/openapi.yaml) ## Scope @@ -5168,14 +5168,3 @@ positional arguments in Tasks 14 and 17 and defined that way in Task 13. `detect class in Tasks 9, 14 and 17. `NormalisedComponent` field names match between Tasks 9, 10, 12, 13 and 17. `ComponentSerializer` field list matches the contract's `Component` properties, asserted by Task 18. - -## Execution Handoff - -Plan complete and saved to `docs/superpowers/plans/2026-08-26-statusboard-backend.md`. Two -execution options: - -**1. Subagent-Driven (recommended)** — a fresh subagent per task, review between tasks, fast -iteration. - -**2. Inline Execution** — execute tasks in this session using executing-plans, batch execution -with checkpoints for review. diff --git a/docs/superpowers/plans/2026-09-03-statusboard-api.md b/docs/plans/2026-09-03-statusboard-api.md similarity index 100% rename from docs/superpowers/plans/2026-09-03-statusboard-api.md rename to docs/plans/2026-09-03-statusboard-api.md From 33dff5d7a987246008e370476519e9bc47244649 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 11:52:43 -0700 Subject: [PATCH 07/94] feat: a search matches a component's whole path Searching a service's name has to reach its components. The vector is weighted so an exact hit on a rollup outranks a leaf that matched only through its service. A reparent rewrites the subtree. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- .../0005_component_search_document.py | 59 +++++++++++++++++++ api/catalog/models.py | 19 ++++++ api/polling/reconcile.py | 28 +++++++++ api/tests/test_catalog_models.py | 48 ++++++++++++++- api/tests/test_reconcile.py | 27 ++++++++- docs/specs/2026-08-23-statusboard-design.md | 1 + 6 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 api/catalog/migrations/0005_component_search_document.py diff --git a/api/catalog/migrations/0005_component_search_document.py b/api/catalog/migrations/0005_component_search_document.py new file mode 100644 index 0000000..8c52a53 --- /dev/null +++ b/api/catalog/migrations/0005_component_search_document.py @@ -0,0 +1,59 @@ +# Generated by Django 5.2.17 on 2026-09-03 18:46 + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +from django.conf import settings +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0004_component_ancestor_ids"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="historicalservicecomponent", + name="search_document", + field=django.contrib.postgres.search.SearchVectorField( + editable=False, null=True + ), + ), + migrations.AddField( + model_name="servicecomponent", + name="search_document", + field=django.contrib.postgres.search.SearchVectorField( + editable=False, null=True + ), + ), + # The column cannot start null. `polling.reconcile` fills it, and + # it runs only for a poller in `PollerQuerySet.active()`. A + # service nobody tracks is never polled again, so its rows would + # stay unfindable by search for good. + # + # `ancestor_ids` is already backfilled by 0004, so the ordered + # walk here only joins the names it names. + migrations.RunSQL( + sql=""" + UPDATE catalog_servicecomponent AS c + SET search_document = + setweight(to_tsvector(coalesce(c.name, '')), 'A') || + setweight(to_tsvector(coalesce(( + SELECT string_agg(a.name, ' ' ORDER BY u.ord) + FROM unnest(c.ancestor_ids) WITH ORDINALITY AS u(id, ord) + JOIN catalog_servicecomponent AS a ON a.id = u.id + ), '')), 'B') || + setweight(to_tsvector(coalesce(s.name, '')), 'C') + FROM catalog_service AS s + WHERE s.id = c.service_id; + """, + reverse_sql=migrations.RunSQL.noop, + ), + migrations.AddIndex( + model_name="servicecomponent", + index=django.contrib.postgres.indexes.GinIndex( + fields=["search_document"], name="components_by_search" + ), + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index 9c24298..bb004c1 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -2,6 +2,7 @@ from django.contrib.postgres.fields import ArrayField from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVectorField from django.core.exceptions import ValidationError from django.db import models from django.db.models.functions import Coalesce @@ -230,6 +231,19 @@ def _descendant_count(): class ServiceComponentQuerySet(models.QuerySet): + def search(self, q): + """Rank a query against the stored path. + + Websearch mode, so several words are an AND. A caller ordering + by anything else drops the rank and keeps the filter. + """ + query = SearchQuery(q, search_type="websearch") + return ( + self.filter(search_document=query) + .annotate(rank=SearchRank(models.F("search_document"), query)) + .order_by("-rank") + ) + def for_display(self, user=None): """Everything `ComponentSerializer` reads, in four queries. @@ -298,6 +312,10 @@ class ServiceComponent(BaseModel): ancestor_ids = ArrayField( models.UUIDField(), default=list, blank=True, editable=False ) + # The whole path, weighted, written by reconcile. Searching a + # service's name must also reach its components. Walking `parent` + # at query time cannot use an index. + search_document = SearchVectorField(null=True, editable=False) objects = ServiceComponentQuerySet.as_manager() status_page_order = models.IntegerField(verbose_name="Page order", default=0) @@ -340,6 +358,7 @@ class Meta(BaseModel.Meta): ] indexes = [ GinIndex(fields=["ancestor_ids"], name="components_by_ancestor"), + GinIndex(fields=["search_document"], name="components_by_search"), ] def live_events(self, kind): diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index 4c32f90..5ee19f9 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -1,5 +1,7 @@ from django.contrib.auth import get_user_model +from django.contrib.postgres.search import SearchVector from django.db import transaction +from django.db.models import Value from django.utils import timezone from catalog.models import ServiceComponent @@ -78,6 +80,7 @@ def _upsert_components(service, components, author): row.updated_by = author row.save(update_fields=["parent", "updated_by"]) rebuild_ancestry(service) + rebuild_search(service) return rows @@ -114,6 +117,31 @@ def chain(node): ServiceComponent.objects.bulk_update(changed, ["ancestor_ids"]) +def rebuild_search(service): + """Write each component's weighted search path. + + `A` is its own name, `B` its ancestors', `C` its service's. An + exact hit on a rollup then outranks a leaf matched only through its + service. + + Ancestor names are joined here, not in SQL. Postgres cannot build a + vector from an array of foreign keys. It must run after + `rebuild_ancestry`, which supplies `ancestor_ids`. + """ + rows = list( + ServiceComponent.objects.filter(service=service).select_related("service") + ) + names = {row.id: row.name for row in rows} + for row in rows: + ancestors = " ".join(names.get(a, "") for a in row.ancestor_ids) + row.search_document = ( + SearchVector(Value(row.name), weight="A") + + SearchVector(Value(ancestors), weight="B") + + SearchVector(Value(row.service.name), weight="C") + ) + ServiceComponent.objects.bulk_update(rows, ["search_document"]) + + def _archive_vanished(service, components, author): seen = {c.external_id for c in components} # A bulk update never reaches `save`, so both columns are written diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 523c18a..90d94a0 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -2,7 +2,7 @@ from django.db.utils import IntegrityError from catalog.models import Service, ServiceComponent, StatusPage -from polling.reconcile import rebuild_ancestry +from polling.reconcile import rebuild_ancestry, rebuild_search from tests.factories import ( ComponentFactory, PollerFactory, @@ -192,3 +192,49 @@ def test_for_display_counts_descendants_without_a_query_a_row( # Two below the top, one below the middle, none below the leaf. assert counts == [0, 1, 2] + + +def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): + # The rollup's own name is an exact, weight-A hit. A leaf matches + # only through its service, at weight C, so it must sort below. + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=parent) + rebuild_ancestry(service) + rebuild_search(service) + + found = list(ServiceComponent.objects.search("twilio")) + assert found[0] == rollup + assert leaf in found + + # OR semantics would let the rollup match too: it says "twilio" but + # never "sms". + assert list(ServiceComponent.objects.search("twilio sms")) == [leaf] + + +def test_search_weights_a_components_own_name_over_its_ancestors_over_its_service(db): + # A single-service fixture cannot separate the weights. Every + # component in it shares one service name, so A and C score the + # same. Three services isolate each weight in turn. + own_service = ServiceFactory(name="Widgets") + own_name = ComponentFactory(service=own_service, name="Twilio") + + ancestor_service = ServiceFactory(name="Gadgets") + ancestor = ComponentFactory(service=ancestor_service, name="Twilio") + via_ancestor = ComponentFactory( + service=ancestor_service, name="Portal", parent=ancestor + ) + rebuild_ancestry(ancestor_service) + rebuild_search(ancestor_service) + + via_service = ServiceFactory(name="Twilio") + via_service_component = ComponentFactory(service=via_service, name="Messaging") + + rebuild_search(own_service) + rebuild_search(via_service) + + ranked = {row.id: row.rank for row in ServiceComponent.objects.search("twilio")} + assert ( + ranked[own_name.id] > ranked[via_ancestor.id] > ranked[via_service_component.id] + ) diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index 822d17a..c410edf 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -7,7 +7,7 @@ NormalisedEvent, NormalisedUpdate, ) -from polling.reconcile import apply_fetch, rebuild_ancestry +from polling.reconcile import apply_fetch, rebuild_ancestry, rebuild_search from status.choices import EventKind, IncidentPhase, Severity, StatusSource from status.models import ComponentStatus, ServiceEvent from tests.factories import ComponentFactory, ServiceFactory @@ -304,3 +304,28 @@ def test_a_loop_in_the_parent_column_does_not_hang_the_pass(): second.refresh_from_db() assert first.ancestor_ids == [second.id] assert second.ancestor_ids == [first.id] + + +@pytest.mark.django_db +def test_a_reparent_rewrites_every_descendants_path(): + # Ancestry and the search vector both carry the chain. A provider + # moving one node must not leave its children searchable under + # where they used to sit. + service = ServiceFactory(name="Acme") + old = ComponentFactory(service=service, name="Legacy", external_id="old") + new = ComponentFactory(service=service, name="Platform", external_id="new") + leaf = ComponentFactory( + service=service, name="Queue", external_id="leaf", parent=old + ) + rebuild_ancestry(service) + rebuild_search(service) + + leaf.parent = new + leaf.save(update_fields=["parent"]) + rebuild_ancestry(service) + rebuild_search(service) + + leaf.refresh_from_db() + assert leaf.ancestor_ids == [new.id] + assert list(ServiceComponent.objects.search("legacy queue")) == [] + assert list(ServiceComponent.objects.search("platform queue")) == [leaf] diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 268add1..5da2b83 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -158,6 +158,7 @@ erDiagram string name uuid parent_id FK "self, null at top level" uuid[] ancestor_ids "root first, written by reconcile" + tsvector search_document "weighted path, written by reconcile" int status_page_order bool is_overall bool is_featured "first key of the suggested sort" From 61e17d7cda5a9106ce68ab3e1d7d98f2535feafd Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 12:18:52 -0700 Subject: [PATCH 08/94] feat: one component collection, four screens Discover, a service's parts, a component's descendants and an event's affected list are one query with different parameters. Four nested routes would have been four copies of it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/filters.py | 33 +++++++ api/catalog/urls.py | 11 +++ api/catalog/views_components.py | 89 +++++++++++++++++ api/tests/test_component_api.py | 169 ++++++++++++++++++++++++++++++++ docs/api/openapi.yaml | 51 ++++++++++ 5 files changed, 353 insertions(+) create mode 100644 api/catalog/views_components.py create mode 100644 api/tests/test_component_api.py diff --git a/api/catalog/filters.py b/api/catalog/filters.py index 86618fb..9fcd2d9 100644 --- a/api/catalog/filters.py +++ b/api/catalog/filters.py @@ -38,11 +38,44 @@ class ComponentFilter(filters.FilterSet): status__severity__lte = filters.NumberFilter( field_name="severity_now", lookup_expr="lte" ) + # The Severity filter offers all six values at once. One exact match + # cannot serve it. + status__severity__in = filters.BaseInFilter( + field_name="severity_now", lookup_expr="in" + ) + service = filters.CharFilter(field_name="service__slug") + # Every descendant, not one level. `parent` would name a query this + # does not run. + ancestor = filters.UUIDFilter(method="filter_ancestor") + event = filters.UUIDFilter(field_name="events__id") + # `for_display` annotates this per viewer. It is not a column, so + # there is nothing to generate the filter from. + is_tracked = filters.BooleanFilter(method="filter_is_tracked") class Meta: model = ServiceComponent fields = {"is_overall": ["exact"]} + def filter_ancestor(self, queryset, name, value): + """Match the stored chain, which the GIN index answers. + + `ancestor_ids` holds every step above a row. A containment test + on it reaches any depth in one comparison. + """ + return queryset.filter(ancestor_ids__contains=[value]) + + def filter_is_tracked(self, queryset, name, value): + """Answer it here, because nobody signed out tracks anything. + + Both directions are true for that reader, so neither is an + error. `for_display` annotates `_is_tracked` as NULL then, and + SQL reads `NOT NULL` as NULL rather than true. Comparing against + the annotation would drop every row from Untracked. + """ + if self.request is None or self.request.user.is_anonymous: + return queryset.none() if value else queryset + return queryset.filter(_is_tracked=value) + class ServiceEventFilter(filters.FilterSet): class Meta: diff --git a/api/catalog/urls.py b/api/catalog/urls.py index 0e6c916..69972c8 100644 --- a/api/catalog/urls.py +++ b/api/catalog/urls.py @@ -2,6 +2,7 @@ from rest_framework.routers import SimpleRouter from catalog.views import CatalogImportView, ServiceViewSet +from catalog.views_components import ComponentDetailView, ComponentListView router = SimpleRouter(trailing_slash=True) # basename "service" gives service-list, service-detail, and one route per @@ -11,5 +12,15 @@ urlpatterns = [ # Above the router so its slug route does not shadow it. path("catalog/import/", CatalogImportView.as_view(), name="catalog-import"), + path( + "catalog/components/", + ComponentListView.as_view(), + name="component-list", + ), + path( + "catalog/components//", + ComponentDetailView.as_view(), + name="component-detail", + ), path("", include(router.urls)), ] diff --git a/api/catalog/views_components.py b/api/catalog/views_components.py new file mode 100644 index 0000000..ec96e9b --- /dev/null +++ b/api/catalog/views_components.py @@ -0,0 +1,89 @@ +"""Every read of a component, in one place. + +Discover, a service's Components tab, a component's descendants and an +event's affected list are one collection with different parameters. Four +nested routes would have been four copies of this queryset. +""" + +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from catalog.filters import ComponentFilter +from catalog.models import ServiceComponent +from catalog.queries import COMPONENT_WATCHER_COUNT +from catalog.serializers import ComponentSerializer +from common.aggregates import StatusAggregateSet +from common.filters import FieldsBackend +from common.ordering import MappedOrderingFilter +from status.queries import CURRENT_SEVERITY + + +class ComponentQueryMixin: + permission_classes = [AllowAny] + serializer_class = ComponentSerializer + # get_queryset reads the request, which schema generation has not + # set. This names the model without running it. + queryset = ServiceComponent.objects.none() + + def get_queryset(self): + return ( + ServiceComponent.objects.for_display(self.request.user) + .annotate( + severity_now=CURRENT_SEVERITY, + watcher_count=COMPONENT_WATCHER_COUNT, + ) + .distinct() + ) + + +class ComponentListView(ComponentQueryMixin, generics.ListAPIView): + aggregate_set = StatusAggregateSet + filterset_class = ComponentFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["name", "status_page_order", "updated_at"] + # Severity ahead of popularity, the same as the service sort this + # replaces. Lower severity is worse, so ascending puts the broken + # first. + SUGGESTED = ["-is_featured", "severity_now", "-watcher_count", "name"] + ordering_map = { + "suggested": SUGGESTED, + "status__severity": ["severity_now"], + } + ordering = SUGGESTED + + def initial(self, request, *args, **kwargs): + """Pick the default sort from what the caller asked for. + + DRF reads `ordering` as an attribute, so a method cannot supply + it. It does not check a default against `ordering_fields` + either, so neither branch below needs an entry there. + """ + super().initial(request, *args, **kwargs) + if request.query_params.get("q"): + # `search` already ranked the rows. A sort applied after it + # throws the ranking away. + self.ordering = ["-rank"] + elif request.query_params.get("service"): + # A service tab reads down the provider's own page. + self.ordering = ["status_page_order"] + + def filter_queryset(self, queryset): + """Rank by relevance when the caller typed something. + + One control and one label, Smart, on every list. A separate + "best match" would name a ranking that does not exist until + somebody types. + """ + q = self.request.query_params.get("q") + if q: + queryset = queryset.search(q) + return super().filter_queryset(queryset) + + +class ComponentDetailView(ComponentQueryMixin, generics.RetrieveAPIView): + filter_backends = [FieldsBackend] + lookup_field = "pk" + # The contract names the path variable `uuid`. DRF renames a `pk` + # one to `id`, which would document a path nothing serves. + lookup_url_kwarg = "uuid" diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py new file mode 100644 index 0000000..3e7a21a --- /dev/null +++ b/api/tests/test_component_api.py @@ -0,0 +1,169 @@ +import pytest +from django.urls import reverse +from django.utils import timezone +from rest_framework.test import APIClient + +from polling.reconcile import rebuild_ancestry, rebuild_search +from status.choices import EventKind, IncidentPhase, Severity, StatusSource +from status.models import ComponentStatus, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +def _reading(component, severity): + """The open status row, which is what `severity_now` reads.""" + return ComponentStatus.objects.create( + component=component, + severity=severity, + source=StatusSource.PROVIDER, + started_at=timezone.now(), + ) + + +@pytest.fixture +def tree(): + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory( + service=service, name="Twilio", is_overall=True, status_page_order=0 + ) + parent = ComponentFactory( + service=service, name="Programmable Messaging", status_page_order=2 + ) + leaf = ComponentFactory( + service=service, name="SMS", parent=parent, status_page_order=1 + ) + rebuild_ancestry(service) + rebuild_search(service) + return service, rollup, parent, leaf + + +def test_the_collection_lists_every_component(client, tree): + # Discover searches all of them, rollups included. Narrowing here + # would make the signed-out board and Discover one list. + response = client.get(reverse("component-list")) + assert response.status_code == 200 + assert response.json()["aggregates"]["total"] == 3 + + +def test_is_overall_narrows_to_one_row_per_service(client, tree): + # This is the signed-out board: one row per service, not per part. + response = client.get(reverse("component-list"), {"is_overall": "true"}) + assert [r["name"] for r in response.json()["results"]] == ["Twilio"] + + +def test_ancestor_returns_every_descendant_not_one_level(client, tree): + # A component's Components tab is the same screen at a different + # root, so it cannot count differently from a service's. + _, _, parent, leaf = tree + response = client.get(reverse("component-list"), {"ancestor": str(parent.id)}) + assert [r["id"] for r in response.json()["results"]] == [str(leaf.id)] + + +def test_q_reaches_a_component_through_its_services_name(client, tree): + # Searching "twilio" must find SMS. Its own name says nothing + # about which service it belongs to. + response = client.get(reverse("component-list"), {"q": "twilio sms"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_service_narrows_to_one_services_parts(client, tree): + # A service's Components tab. The rollup is excluded there, + # because the header already carries the service's status. + service, _, _, _ = tree + response = client.get( + reverse("component-list"), {"service": service.slug, "is_overall": "false"} + ) + assert {r["name"] for r in response.json()["results"]} == { + "Programmable Messaging", + "SMS", + } + + +def test_severity_in_takes_several_values(client, tree): + # The Severity filter offers all six values at once, so a single + # exact match cannot serve it. Undeclared, django-filter drops the + # parameter without a word and the operational row comes back too. + _, rollup, parent, leaf = tree + _reading(rollup, Severity.MAJOR_OUTAGE) + _reading(parent, Severity.DEGRADED) + _reading(leaf, Severity.OPERATIONAL) + response = client.get(reverse("component-list"), {"status__severity__in": "0,1,2"}) + assert {r["name"] for r in response.json()["results"]} == { + "Twilio", + "Programmable Messaging", + } + + +def test_the_detail_answers_by_uuid(client, tree): + _, _, _, leaf = tree + response = client.get(reverse("component-detail", args=[leaf.id])) + assert response.status_code == 200 + assert response.json()["name"] == "SMS" + + +def test_a_query_keeps_the_rank_the_search_worked_out(client, tree): + # The rollup carries the word in its own name. The other two match + # only through their service. A default sort applied after the + # search discards the rank, and then the list reads alphabetically. + response = client.get(reverse("component-list"), {"q": "twilio"}) + assert response.json()["results"][0]["name"] == "Twilio" + + +def test_a_services_parts_keep_the_status_pages_own_order(client, tree): + # The service tab shows the components in the order the provider + # lists them. The suggested sort would put them in name order. + service, _, _, _ = tree + response = client.get( + reverse("component-list"), {"service": service.slug, "is_overall": "false"} + ) + assert [r["name"] for r in response.json()["results"]] == [ + "SMS", + "Programmable Messaging", + ] + + +def test_is_tracked_narrows_to_what_the_viewer_watches(tree): + # Showing All, Tracked or Untracked. The flag is per viewer, so the + # filter has to read the annotation and not a column. + _, _, _, leaf = tree + item = track(leaf) + viewer = APIClient() + viewer.force_authenticate(item.dashboard.owner) + response = viewer.get(reverse("component-list"), {"is_tracked": "true"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_signed_out_nothing_is_tracked(client, tree): + # Nobody signed out tracks anything, so Tracked is empty for them. + response = client.get(reverse("component-list"), {"is_tracked": "true"}) + assert response.json()["results"] == [] + + +def test_signed_out_everything_is_untracked(client, tree): + # The same fact makes Untracked the whole list. Comparing against + # the annotation instead reads `NOT NULL`, which SQL answers NULL, + # and Untracked renders empty for a signed-out reader. + response = client.get(reverse("component-list"), {"is_tracked": "false"}) + assert {r["name"] for r in response.json()["results"]} == { + "Twilio", + "Programmable Messaging", + "SMS", + } + + +def test_event_narrows_to_the_components_one_event_affects(client, tree): + # An event's affected list. Only the components the provider names + # on that event belong on it. + service, _, parent, _ = tree + event = ServiceEvent.objects.create( + service=service, + external_id="inc-1", + kind=EventKind.INCIDENT, + title="Messages are delayed", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + event.affected_components.add(parent) + response = client.get(reverse("component-list"), {"event": str(event.id)}) + assert [r["name"] for r in response.json()["results"]] == ["Programmable Messaging"] diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 7dc661d..8c6ea96 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -246,6 +246,57 @@ paths: description: No supported status page found at that URL content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } + /catalog/components/: + get: + tags: [catalog] + summary: Search and list components + description: | + One collection, four screens. Discover with no parameters, a service's parts with + `service`, a component's descendants with `ancestor`, an event's affected list with + `event`. + + `q` matches a component's whole path, so `twilio` finds `SMS`. Results are then ranked by + relevance. Without `q` the default is `suggested`, except under `service`, where it is + `status_page_order`. + security: [{}, { bearer: [] }] + parameters: + - { name: q, in: query, schema: { type: string } } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: ancestor, in: query, schema: { type: string, format: uuid }, description: Every descendant, at any depth } + - { name: event, in: query, schema: { type: string, format: uuid }, description: The components this event affects } + - { name: is_overall, in: query, schema: { type: boolean }, description: '`true` is one row per service, which is signed-out Home.' } + - { name: is_tracked, in: query, schema: { type: boolean }, description: 'Authenticated only. Drives Showing All / Tracked / Untracked.' } + - { name: status__severity, in: query, schema: { type: integer, minimum: 0, maximum: 5 } } + - { name: status__severity__lte, in: query, schema: { type: integer, minimum: 0, maximum: 5 } } + - { name: status__severity__in, in: query, schema: { type: string }, description: 'Comma separated, for a Severity filter offering every value at once.' } + - { name: ordering, in: query, schema: { type: string, enum: [suggested, status_page_order, -status_page_order, status__severity, -status__severity, name, -name, updated_at, -updated_at], default: suggested } } + - { $ref: '#/components/parameters/PageSize' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/StatusAggregates' } + results: { type: array, items: { $ref: '#/components/schemas/Component' } } + + /catalog/components/{uuid}/: + get: + tags: [catalog] + summary: Component detail + security: [{}, { bearer: [] }] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } + '404': { $ref: '#/components/responses/Error' } /catalog/services/{slug}/: get: From 3fa9a619da6a00fe304d04dcdc24c5cd7786b7ca Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 12:38:44 -0700 Subject: [PATCH 09/94] feat: an event records who found it A provider claims an event we opened, which fills in their id. That destroys the fact that we saw the outage first, so the origin needs a column. An update records its own author, so one card can hold both. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/common/views.py | 9 ++- api/status/choices.py | 14 +++++ .../migrations/0003_event_detected_by.py | 55 +++++++++++++++++++ api/status/models.py | 31 ++++++++++- api/status/serializers.py | 2 +- api/tests/test_meta.py | 1 + api/tests/test_status_models.py | 40 ++++++++++++++ docs/api/openapi.yaml | 28 +++++++--- docs/check_docs.py | 1 + docs/specs/2026-08-23-statusboard-design.md | 4 +- 10 files changed, 172 insertions(+), 13 deletions(-) create mode 100644 api/status/migrations/0003_event_detected_by.py diff --git a/api/common/views.py b/api/common/views.py index 9f59509..ac5f47c 100644 --- a/api/common/views.py +++ b/api/common/views.py @@ -9,7 +9,13 @@ from catalog.choices import StatusPageProvider from common.serializers import MetaSerializer -from status.choices import EVENT_PHASES_BY_KIND, EventKind, Severity, StatusSource +from status.choices import ( + EVENT_PHASES_BY_KIND, + EventKind, + EventSource, + Severity, + StatusSource, +) def _labels(choices): @@ -52,6 +58,7 @@ def get(self, request): "status_source": _labels(StatusSource), "status_page_provider": _labels(StatusPageProvider), "event_kind": _labels(EventKind), + "event_source": _labels(EventSource), "event_phase": { str(kind): _labels(phases) for kind, phases in EVENT_PHASES_BY_KIND.items() diff --git a/api/status/choices.py b/api/status/choices.py index 53dae77..e216ff6 100644 --- a/api/status/choices.py +++ b/api/status/choices.py @@ -28,6 +28,9 @@ class EventKind(models.TextChoices): class IncidentPhase(models.TextChoices): + # First, because it precedes anything a provider posts. We write it + # when a severity drops with nothing explaining it. + DETECTED = "detected", "Detected" INVESTIGATING = "investigating", "Investigating" IDENTIFIED = "identified", "Identified" MONITORING = "monitoring", "Monitoring" @@ -41,6 +44,17 @@ class MaintenancePhase(models.TextChoices): COMPLETED = "completed", "Completed" +class EventSource(models.TextChoices): + """Who produced a row: the provider's page, or this deployment. + + An event records who opened it. An update records who wrote it. A + claimed event has both, one per update. + """ + + PROVIDER = "provider", "Provider" + SYSTEM = "system", "Statusboard" + + EVENT_PHASES_BY_KIND = { EventKind.INCIDENT: IncidentPhase, EventKind.MAINTENANCE: MaintenancePhase, diff --git a/api/status/migrations/0003_event_detected_by.py b/api/status/migrations/0003_event_detected_by.py new file mode 100644 index 0000000..b9266d0 --- /dev/null +++ b/api/status/migrations/0003_event_detected_by.py @@ -0,0 +1,55 @@ +# Generated by Django 5.2.17 on 2026-09-03 19:35 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0005_component_search_document"), + ("polling", "0002_every_service_has_a_poller"), + ("status", "0002_events_by_service_and_kind"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="serviceevent", + name="one_event_per_provider_id", + ), + migrations.AddField( + model_name="eventupdate", + name="source", + field=models.CharField( + choices=[("provider", "Provider"), ("system", "Statusboard")], + db_default="provider", + default="provider", + max_length=32, + ), + ), + migrations.AddField( + model_name="serviceevent", + name="detected_by", + field=models.CharField( + choices=[("provider", "Provider"), ("system", "Statusboard")], + db_default="provider", + default="provider", + max_length=32, + ), + ), + migrations.AlterField( + model_name="serviceevent", + name="external_id", + field=models.CharField( + blank=True, max_length=200, null=True, verbose_name="Provider ID" + ), + ), + migrations.AddConstraint( + model_name="serviceevent", + constraint=models.UniqueConstraint( + condition=models.Q(("external_id__isnull", False)), + fields=("service", "external_id"), + name="one_event_per_provider_id", + ), + ), + ] diff --git a/api/status/models.py b/api/status/models.py index f9595ea..aded507 100644 --- a/api/status/models.py +++ b/api/status/models.py @@ -9,6 +9,7 @@ CLOSED_PHASES, EVENT_PHASES_BY_KIND, EventKind, + EventSource, Severity, StatusSource, ) @@ -124,7 +125,20 @@ class ServiceEvent(BaseModel): service = models.ForeignKey( Service, on_delete=models.CASCADE, related_name="events" ) - external_id = models.CharField(verbose_name="Provider ID", max_length=200) + # Null until a provider claims this event. We open events a + # provider never published, and they have no id of theirs. + external_id = models.CharField( + verbose_name="Provider ID", max_length=200, null=True, blank=True + ) + # Who opened it, never rewritten by a claim. `external_id IS NULL` + # cannot answer this: claiming fills the column in, which destroys + # the fact that we found the outage first. + detected_by = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) kind = models.CharField(max_length=32, choices=EventKind.choices) title = models.CharField(max_length=500) phase = models.CharField(max_length=32) @@ -154,8 +168,13 @@ class ServiceEvent(BaseModel): class Meta(BaseModel.Meta): constraints = [ + # Partial. Two events we opened both hold null, and null + # does not collide with null in Postgres anyway. Stating + # the condition says the exemption is deliberate. models.UniqueConstraint( - fields=["service", "external_id"], name="one_event_per_provider_id" + fields=["service", "external_id"], + condition=models.Q(external_id__isnull=False), + name="one_event_per_provider_id", ) ] indexes = [ @@ -202,6 +221,14 @@ class EventUpdate(BaseModel): phase = models.CharField(max_length=32) body = models.TextField() posted_at = models.DateTimeField(verbose_name="Posted") + # Who wrote this post. A claimed event holds both: our detection + # first, then the provider's log. + source = models.CharField( + max_length=32, + choices=EventSource.choices, + default=EventSource.PROVIDER, + db_default=EventSource.PROVIDER, + ) def clean(self): """A phase belongs to one kind, the same rule the event follows. diff --git a/api/status/serializers.py b/api/status/serializers.py index f62c1b1..6def9cf 100644 --- a/api/status/serializers.py +++ b/api/status/serializers.py @@ -32,7 +32,7 @@ class Meta: class EventUpdateSerializer(FieldsMixin, serializers.ModelSerializer): class Meta: model = EventUpdate - fields = ["phase", "body", "posted_at"] + fields = ["phase", "body", "posted_at", "source"] class ServiceEventSerializer(FieldsMixin, serializers.ModelSerializer): diff --git a/api/tests/test_meta.py b/api/tests/test_meta.py index bc2be8e..d9f0375 100644 --- a/api/tests/test_meta.py +++ b/api/tests/test_meta.py @@ -36,6 +36,7 @@ def test_meta_publishes_every_enum(): "status_source", "status_page_provider", "event_kind", + "event_source", "event_phase", } assert body["enums"]["severity"]["0"] == "Major outage" diff --git a/api/tests/test_status_models.py b/api/tests/test_status_models.py index 04c624b..b57d3f3 100644 --- a/api/tests/test_status_models.py +++ b/api/tests/test_status_models.py @@ -4,7 +4,9 @@ from django.utils import timezone from status.choices import ( + CLOSED_PHASES, EventKind, + EventSource, IncidentPhase, MaintenancePhase, Severity, @@ -100,3 +102,41 @@ def test_an_event_may_name_no_component_at_all(): starts_at=timezone.now(), ) assert event.affected_components.count() == 0 + + +def test_two_system_events_can_sit_on_one_service(db): + # Neither has a provider id. The unique key is partial, so a null + # does not collide with another null. + service = ServiceFactory() + for _ in range(2): + ServiceEvent.objects.create( + service=service, + external_id=None, + kind=EventKind.INCIDENT, + title="Degraded", + phase=IncidentPhase.DETECTED, + detected_by=EventSource.SYSTEM, + starts_at=timezone.now(), + ) + assert ServiceEvent.objects.filter(service=service).count() == 2 + + +def test_a_provider_id_is_still_unique_per_service(db): + # A second poll of the same page must update the row, never add one. + service = ServiceFactory() + fields = { + "service": service, + "kind": EventKind.INCIDENT, + "title": "Outage", + "phase": IncidentPhase.INVESTIGATING, + "starts_at": timezone.now(), + } + ServiceEvent.objects.create(external_id="abc", **fields) + with pytest.raises(IntegrityError): + ServiceEvent.objects.create(external_id="abc", **fields) + + +def test_detected_is_an_open_phase(db): + # A detected event is running. Listing it as closed would hide + # every outage no provider ever wrote up. + assert IncidentPhase.DETECTED not in CLOSED_PHASES diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 8c6ea96..0a7fb3f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -536,7 +536,11 @@ components: additionalProperties: { type: string } description: | Both sets, keyed by kind: - `{"incident": {"investigating": "Investigating", …}, "maintenance": {"scheduled": "Scheduled", …}}` + `{"incident": {"detected": "Detected", …}, "maintenance": {"scheduled": "Scheduled", …}}` + event_source: + type: object + additionalProperties: { type: string } + description: '{"provider": "Provider", "system": "Statusboard"}' TokenPair: type: object properties: @@ -926,7 +930,7 @@ components: phase: type: string description: | - `investigating, identified, monitoring, resolved` for an incident. + `detected, investigating, identified, monitoring, resolved` for an incident. `scheduled, in_progress, verifying, completed` for maintenance. Render the label from `/meta/` `enums.event_phase`, keyed by `kind`. @@ -934,12 +938,20 @@ components: ends_at: { type: string, format: date-time, nullable: true, description: When it resolved, or when the window closes. } updates: type: array - items: - type: object - properties: - phase: { type: string } - body: { type: string } - posted_at: { type: string, format: date-time } + items: { $ref: '#/components/schemas/EventUpdate' } + EventUpdate: + type: object + description: One posted update within an event's timeline. + properties: + phase: { type: string } + body: { type: string } + posted_at: { type: string, format: date-time } + source: + type: string + enum: [provider, system] + description: | + Who wrote this update. `system` is a severity change we recorded ourselves. + Render the label from `/meta/` `enums.event_source`. ServiceRef: type: object properties: diff --git a/docs/check_docs.py b/docs/check_docs.py index a3590e2..e551369 100755 --- a/docs/check_docs.py +++ b/docs/check_docs.py @@ -35,6 +35,7 @@ "Status": "ComponentStatus", "ServiceEvent": "ServiceEvent", "EventRef": "ServiceEvent", + "EventUpdate": "EventUpdate", "Me": "User", } PLAIN = { diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 5da2b83..9086ec8 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -175,7 +175,8 @@ erDiagram ServiceEvent { uuid id PK uuid service_id FK - string external_id "unique with service" + string external_id "provider's id, null until claimed" + enum detected_by "provider or system, never rewritten" enum kind "incident or maintenance" string title enum phase "valid set depends on kind" @@ -188,6 +189,7 @@ erDiagram enum phase "TextChoices" text body "written by the provider" datetime posted_at + enum source "provider or system" } PollRun { uuid id PK From db864205e0843e78756d97227e3e0b1b1f33d098 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 12:52:27 -0700 Subject: [PATCH 10/94] fix: /meta/ honours ?fields=, including dotted paths into enums MetaView built its response dict by hand and returned it directly, skipping MetaSerializer entirely. FieldsMixin prunes at construction, so ?fields= was declared on /meta/ but did nothing, on every variant. This let a client ask for one field and always get the whole payload. MetaSerializer now carries FieldsMixin, and MetaView serializes through it, matching the pattern MeView already uses. enums stays a plain dict built once, in MetaView, but is now read through a SerializerMethodField that filters it by the branch FieldsMixin already parses under "enums." A nested FieldsMixin serializer was ruled out: it would restate the enum list a second time, and one definition drifts from the other when an enum is added. The contract-versus-code suite asserted only that ?fields= was declared on every GET, which is how this shipped unnoticed. It now also asserts, driven from openapi.yaml's own path list, that ?fields= actually prunes each documented GET's response. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/common/serializers.py | 26 +++++++- api/common/views.py | 37 +++++------ api/tests/test_contract.py | 123 +++++++++++++++++++++++++++++++++++++ api/tests/test_meta.py | 49 +++++++++++++++ 4 files changed, 214 insertions(+), 21 deletions(-) diff --git a/api/common/serializers.py b/api/common/serializers.py index 43cfb22..272d0d8 100644 --- a/api/common/serializers.py +++ b/api/common/serializers.py @@ -1,7 +1,10 @@ from enum import StrEnum +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers +from common.mixins import FieldsMixin + class ErrorCode(StrEnum): """What a failure calls itself. @@ -25,13 +28,30 @@ class ErrorSerializer(serializers.Serializer): detail = serializers.CharField() -class MetaSerializer(serializers.Serializer): +class MetaSerializer(FieldsMixin, serializers.Serializer): """Deployment-wide configuration. Nothing here is a user preference.""" poll_interval_seconds = serializers.IntegerField() poll_cooldown_seconds = serializers.IntegerField() default_page_size = serializers.IntegerField() max_page_size = serializers.IntegerField() - enums = serializers.DictField( - help_text="Label maps for every enum, keyed by enum name." + enums = serializers.SerializerMethodField() + + @extend_schema_field( + serializers.DictField( + help_text="Label maps for every enum, keyed by enum name." + ) ) + def get_enums(self, obj): + """Keep only the enums a dotted `?fields=enums.` names. + + `enums` is a plain dict, not a nested serializer, so + `FieldsMixin` cannot prune inside it on its own. This reads the + branch under "enums" and filters by hand instead. `MetaView` + still names every enum in the one place it always has. + """ + enums = obj["enums"] + wanted = self.child_tree("enums") + if not wanted: + return enums + return {name: labels for name, labels in enums.items() if name in wanted} diff --git a/api/common/views.py b/api/common/views.py index ac5f47c..0107c39 100644 --- a/api/common/views.py +++ b/api/common/views.py @@ -47,25 +47,26 @@ class MetaView(APIView): @extend_schema(responses={200: MetaSerializer}) def get(self, request): - return Response( - { - "poll_interval_seconds": settings.POLL_INTERVAL_SECONDS, - "poll_cooldown_seconds": settings.POLL_COOLDOWN_SECONDS, - "default_page_size": settings.DEFAULT_PAGE_SIZE, - "max_page_size": settings.MAX_PAGE_SIZE, - "enums": { - "severity": _labels(Severity), - "status_source": _labels(StatusSource), - "status_page_provider": _labels(StatusPageProvider), - "event_kind": _labels(EventKind), - "event_source": _labels(EventSource), - "event_phase": { - str(kind): _labels(phases) - for kind, phases in EVENT_PHASES_BY_KIND.items() - }, + data = { + "poll_interval_seconds": settings.POLL_INTERVAL_SECONDS, + "poll_cooldown_seconds": settings.POLL_COOLDOWN_SECONDS, + "default_page_size": settings.DEFAULT_PAGE_SIZE, + "max_page_size": settings.MAX_PAGE_SIZE, + "enums": { + "severity": _labels(Severity), + "status_source": _labels(StatusSource), + "status_page_provider": _labels(StatusPageProvider), + "event_kind": _labels(EventKind), + "event_source": _labels(EventSource), + "event_phase": { + str(kind): _labels(phases) + for kind, phases in EVENT_PHASES_BY_KIND.items() }, - } - ) + }, + } + # `Response(data)` skips `MetaSerializer`, so `?fields=` never + # pruned. `FieldsMixin` prunes at construction, on the instance. + return Response(MetaSerializer(data, context={"request": request}).data) class ApiDocsView(TemplateView): diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py index 52e9a91..f1f46e3 100644 --- a/api/tests/test_contract.py +++ b/api/tests/test_contract.py @@ -3,8 +3,17 @@ import pytest import yaml from django.urls import reverse +from django.utils import timezone from drf_spectacular.drainage import GENERATOR_STATS, reset_generator_stats from drf_spectacular.generators import SchemaGenerator +from rest_framework.test import APIClient +from rest_framework_simplejwt.tokens import RefreshToken + +from authentication.models import User +from dashboards.models import DashboardItem +from status.choices import EventKind, IncidentPhase +from status.models import ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, StatusPageFactory CONTRACT = Path(__file__).resolve().parents[2] / "docs" / "api" / "openapi.yaml" @@ -49,6 +58,120 @@ def test_every_endpoint_accepts_the_fields_parameter(generated): assert "fields" in names, f"{verb.upper()} {path} does not accept ?fields=" +def _authed_client(): + """A client bearing a fresh user's access token.""" + user = User.objects.create( + email=f"fields-probe-{User.objects.count()}@example.test" + ) + api = APIClient() + api.credentials( + HTTP_AUTHORIZATION=f"Bearer {RefreshToken.for_user(user).access_token}" + ) + return api, user + + +def _meta_probe(): + return APIClient(), reverse("meta"), "max_page_size" + + +def _me_probe(): + api, _user = _authed_client() + return api, reverse("me"), "email" + + +def _service_list_probe(): + service = ServiceFactory() + StatusPageFactory(service=service) + ComponentFactory(service=service, is_overall=True) + return APIClient(), reverse("service-list"), "id" + + +def _service_detail_probe(): + service = ServiceFactory() + StatusPageFactory(service=service) + ComponentFactory(service=service, is_overall=True) + return APIClient(), reverse("service-detail", kwargs={"slug": service.slug}), "id" + + +def _service_components_probe(): + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + url = reverse("service-components", kwargs={"slug": service.slug}) + return APIClient(), url, "id" + + +def _service_events_probe(): + service = ServiceFactory() + ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + url = reverse("service-events", kwargs={"slug": service.slug}) + return APIClient(), url, "id" + + +def _component_list_probe(): + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + return APIClient(), reverse("component-list"), "id" + + +def _component_detail_probe(): + service = ServiceFactory() + component = ComponentFactory(service=service, is_overall=True) + url = reverse("component-detail", kwargs={"uuid": component.id}) + return APIClient(), url, "id" + + +def _board_components_probe(): + api, user = _authed_client() + board = user.default_dashboard + service = ServiceFactory() + component = ComponentFactory(service=service, is_overall=True) + DashboardItem.objects.create(dashboard=board, component=component) + url = reverse("board-components", kwargs={"uuid": board.id}) + return api, url, "id" + + +# One builder per documented GET path: a client, a live URL, and a field +# to keep. `/events/` and its children are not here. They are not in +# the contract yet, so this test does not cover them until they land. +FIELDS_PROBES = { + "/meta/": _meta_probe, + "/me/": _me_probe, + "/catalog/services/": _service_list_probe, + "/catalog/services/{slug}/": _service_detail_probe, + "/catalog/services/{slug}/components/": _service_components_probe, + "/catalog/services/{slug}/events/": _service_events_probe, + "/catalog/components/": _component_list_probe, + "/catalog/components/{uuid}/": _component_detail_probe, + "/dashboards/{uuid}/components/": _board_components_probe, +} + + +def test_the_fields_probes_cover_every_documented_get(committed): + # A GET added later and left out here would ship unpruned and + # nothing would say so, the way /meta/ did. + documented = {path for path, ops in committed["paths"].items() if "get" in ops} + assert documented == set(FIELDS_PROBES) + + +@pytest.mark.django_db +@pytest.mark.parametrize("path", sorted(FIELDS_PROBES)) +def test_fields_prunes_the_response(path): + # Declaring ?fields= on a view is not the same as honouring it. + # /meta/ built its body by hand and never ran it through the + # serializer that prunes, so the parameter did nothing. + client, url, field = FIELDS_PROBES[path]() + body = client.get(url, {"fields": field}).json() + kept = body["results"][0] if isinstance(body, dict) and "results" in body else body + assert set(kept) == {field}, f"GET {path} ignored ?fields=: {sorted(kept)}" + + @pytest.mark.django_db def test_the_component_schema_has_exactly_the_documented_fields(committed): from catalog.serializers import ComponentSerializer diff --git a/api/tests/test_meta.py b/api/tests/test_meta.py index d9f0375..7f64536 100644 --- a/api/tests/test_meta.py +++ b/api/tests/test_meta.py @@ -47,6 +47,55 @@ def test_meta_publishes_every_enum(): assert "scheduled" in body["enums"]["event_phase"]["maintenance"] +@pytest.mark.django_db +def test_fields_prunes_the_top_level(): + # /meta/ built its dict and skipped the serializer that prunes it. + # ?fields= was declared but did nothing. + body = APIClient().get(reverse("meta") + "?fields=max_page_size").json() + assert body == {"max_page_size": 200} + + +@pytest.mark.django_db +def test_fields_prunes_inside_enums(): + # `enums` is a plain dict. FieldsMixin only prunes a nested + # serializer, so a dotted path into `enums` reached nothing. + url = reverse("meta") + "?fields=enums.status_source" + body = APIClient().get(url).json() + assert body == { + "enums": { + "status_source": { + "provider": "Provider", + "components": "Components", + "incidents": "Incidents", + } + } + } + + +@pytest.mark.django_db +def test_fields_enums_with_no_dotted_path_keeps_every_enum(): + # `?fields=enums` names the whole map, not one entry in it. + body = APIClient().get(reverse("meta") + "?fields=enums").json() + assert set(body) == {"enums"} + assert set(body["enums"]) == { + "severity", + "status_source", + "status_page_provider", + "event_kind", + "event_source", + "event_phase", + } + + +@pytest.mark.django_db +def test_fields_keeps_two_dotted_paths_under_enums(): + # Two dotted paths under enums must both survive. Parsing only the + # last one would silently drop status_source here. + url = reverse("meta") + "?fields=enums.status_source,enums.event_kind" + body = APIClient().get(url).json() + assert set(body["enums"]) == {"status_source", "event_kind"} + + @pytest.mark.django_db def test_meta_publishes_the_deployment_defaults(): body = APIClient().get(reverse("meta")).json() From 98efdfe391c88e477c3c3382122d6dda496781f7 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 13:10:01 -0700 Subject: [PATCH 11/94] fix: a code span counts as one word in the prose checker check_prose.py's words() dropped only tokens starting with a backtick. A multi-word span like `external_id IS NULL` leaked its middle and closing tokens as prose, while a single-word span like `field` was dropped entirely and undercounted. Neither matched "one idea per sentence": a code span is one thing to read, not a handful of words to weigh individually or discard. words() now collapses a balanced code span to one token before counting. An unmatched backtick falls back to the old per-token filter rather than raising, since sentences() can in principle hand it half a span. Fixing the count newly flagged eight sentences across api/; each is reworded here to fit, without reflowing prose that was already legal. AGENTS.md now says "20 words or fewer" rather than "Under 20 words": that has always been the enforced rule, and 44 already-reviewed sentences sit at exactly 20. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 2 +- api/api/defaults.py | 11 +++--- api/catalog/serializers.py | 4 +- api/catalog/views.py | 4 +- api/common/errors.py | 6 +-- api/status/models.py | 2 +- api/tests/test_catalog_models.py | 2 +- api/tests/test_prose_checker.py | 66 ++++++++++++++++++++++++++++++++ bin/check_prose.py | 12 +++++- 9 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 api/tests/test_prose_checker.py diff --git a/AGENTS.md b/AGENTS.md index dd606b3..f7d5f6c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ Comments, docstrings, commit messages and documentation follow ASD-STE100. -- One idea per sentence. Under 20 words. +- One idea per sentence. 20 words or fewer. - Active voice. Present tense. - No em dashes. A qualifier goes in brackets: `GitHub (2026-08-30 15:32)`. - A standalone `—` is allowed as an empty-cell marker. It is a placeholder, diff --git a/api/api/defaults.py b/api/api/defaults.py index ee6b2f0..b7d35e0 100644 --- a/api/api/defaults.py +++ b/api/api/defaults.py @@ -69,18 +69,17 @@ def parse(cls, value): ) from error -# Which deployment this is. We named it, so it is resolved here, and -# `debug` and `secret_key` in `settings.py` read it rather than being -# handed it. +# Which deployment this is. We named it, so `debug` and `secret_key` +# in `settings.py` read it here rather than being handed it. ENVIRONMENT = Environment.parse(os.environ.get("ENVIRONMENT")) class Throttle(StrEnum): """A rate this deployment enforces. - A view names its scope and `settings.py` gives it a rate, so the - string was written twice and had to match. DRF names the first two; - we named the rest. + A view names its scope and `settings.py` gives it a rate. The two + spellings had to match. DRF names the first two; we named the + rest. """ ANONYMOUS = "anon" diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index 66360c3..cf0515c 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -202,8 +202,8 @@ def get_tracked_component_count(self, service): class ImportRequestSerializer(serializers.Serializer): """The body of POST /catalog/import/. - Named `status_page_url`, not `url`, because that is what the contract - documents and the contract is what the client is written against. + Named `status_page_url`, not `url`. The contract names it that way, + and the client follows the contract. """ status_page_url = serializers.URLField() diff --git a/api/catalog/views.py b/api/catalog/views.py index 0f2398f..853bfc0 100644 --- a/api/catalog/views.py +++ b/api/catalog/views.py @@ -86,8 +86,8 @@ class ServiceViewSet(ReadOnlyModelViewSet): def get_queryset(self): # `q` is SearchFilter's, through SEARCH_PARAM. It was a hand - # rolled `name__icontains` beside a `search_fields` that no - # backend read, so the slug was declared searchable and was not. + # rolled `name__icontains` beside a `search_fields` no backend + # read. The slug was declared searchable, and was not. return Service.objects.for_display(self.request.user).annotate( watcher_count=WATCHER_COUNT, severity_now=OVERALL_SEVERITY, diff --git a/api/common/errors.py b/api/common/errors.py index b27cdae..68c152a 100644 --- a/api/common/errors.py +++ b/api/common/errors.py @@ -2,7 +2,7 @@ `ErrorSerializer` and the codes were declared and documented from the start, and nothing raised them. Three different shapes came back -instead: DRF's `detail`, a field map from a serializer, and an +instead. DRF's `detail`, a field map from a serializer, and an uncaught exception as a 500. A client is generated from the contract, so it looks for `code` and @@ -49,8 +49,8 @@ def _code(exception): return exception.code if isinstance(exception, Throttled): return ErrorCode.THROTTLED - # Django raises its own 404 and DRF translates it, so the exception - # reaching here is `Http404` rather than DRF's `NotFound`. + # Django raises its own 404, and DRF translates it. The exception + # reaching here is `Http404`, not DRF's `NotFound`. if isinstance(exception, NotFound | Http404): return ErrorCode.NOT_FOUND return None diff --git a/api/status/models.py b/api/status/models.py index aded507..a07b3e3 100644 --- a/api/status/models.py +++ b/api/status/models.py @@ -178,7 +178,7 @@ class Meta(BaseModel.Meta): ) ] indexes = [ - # `live` reads this for every component on a page, and the + # `live` reads this for every component on a page. The # only index was the unique key, which serves neither # column. Ordered as the filter narrows: the service first, # then the kind, then the phase it excludes. diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 90d94a0..65b6673 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -120,7 +120,7 @@ def test_a_name_with_no_slug_characters_still_gets_one(): @pytest.mark.django_db def test_archiving_sets_the_date(): - # `updated_at` cannot answer when a provider dropped a component: it + # `updated_at` cannot answer when a provider dropped a component. It # moves on every save, so a later rename would erase it. component = ComponentFactory() assert component.archived_at is None diff --git a/api/tests/test_prose_checker.py b/api/tests/test_prose_checker.py new file mode 100644 index 0000000..7deda20 --- /dev/null +++ b/api/tests/test_prose_checker.py @@ -0,0 +1,66 @@ +"""Behaviour tests for `bin/check_prose.py`. + +`bin/` has no test runner of its own, and the only one in this repo is +this Django suite. `check_prose.py` needs no Django, so it is loaded +by path rather than inventing a second runner for one script. +""" + +import importlib.util +from pathlib import Path + +CHECK_PROSE = Path(__file__).resolve().parents[2] / "bin" / "check_prose.py" + + +def _load_check_prose(): + spec = importlib.util.spec_from_file_location("check_prose", CHECK_PROSE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +check_prose = _load_check_prose() + + +def test_a_multi_word_code_span_counts_as_one_word(): + # The old filter dropped only the opening token. A five-word span + # leaked four words into the count instead of naming one. + with_span = check_prose.words("Reject the row unless `external_id IS NULL` matches") + without_span = check_prose.words("Reject the row unless matches") + assert with_span == without_span + 1 + + +def test_a_single_token_code_span_still_counts_as_one_word(): + # The old filter dropped a lone backtick-prefixed token outright, + # so a sentence naming one field was undercounted by one. + assert check_prose.words( + "Reject the row unless `external_id` matches" + ) == check_prose.words("Reject the row unless `external_id IS NULL` matches") + + +def test_a_code_span_followed_by_a_comma_counts_only_the_span(): + # A padded substitution once tore the comma into its own token, + # counting a span and its punctuation as two words. + assert check_prose.words("The `foo`, which is set, wins") == 6 + + +def test_a_code_span_followed_by_a_full_stop_counts_only_the_span(): + # Same bug at a sentence end. The period must not become a free + # word once the span collapses. + assert check_prose.words("Set by `foo`. Then read") == 5 + + +def test_an_unmatched_backtick_does_not_raise(): + # sentences() can in principle hand words() a fragment holding half + # a span. That must fall back to counting, not crash the checker. + assert check_prose.words("A stray ` mark stays here") == 5 + + +def test_a_twenty_word_sentence_is_not_flagged(): + sentence = " ".join(["word"] * 20) + assert not check_prose.prose_problems([(sentence, 1)]) + + +def test_a_twenty_one_word_sentence_is_flagged(): + sentence = " ".join(["word"] * 21) + problems = check_prose.prose_problems([(sentence, 1)]) + assert [rule for _line, rule, _detail in problems] == ["long"] diff --git a/bin/check_prose.py b/bin/check_prose.py index 46f913c..b750f9d 100755 --- a/bin/check_prose.py +++ b/bin/check_prose.py @@ -28,6 +28,8 @@ # A directive is not prose. DIRECTIVE = re.compile(r"^#\s*(noqa|type:|ruff:|pragma:|fmt:|mypy:|!)") CODE_ISH = re.compile(r"^\s*[\w.]+\([^)]*\)\s*$|^\s*[-*]\s|^\s{4,}\S") +# A code span, however many tokens inside, is one thing to read. +CODE_SPAN = re.compile(r"`[^`]*`") def sentences(blocks): @@ -52,7 +54,15 @@ def sentences(blocks): def words(sentence): - """Count words, ignoring anything that is code rather than prose.""" + """Count words. A code span counts as one, however many it wraps. + + A span like `external_id IS NULL` is one thing to read, not three + words to weigh. A lone backtick has no partner to pair into a + span. It falls back to the old per-token filter, instead of + raising. + """ + if sentence.count("`") % 2 == 0: + sentence = CODE_SPAN.sub("codespan", sentence) countable = [ w for w in sentence.split() From 062632451649b17b914633c75235387374b6749a Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 14:09:09 -0700 Subject: [PATCH 12/94] feat: a status change with no provider post becomes an event A provider can move a component to Degraded and never write an incident. An event-only feed hid those, and the closed status span was exposed nowhere else. The client now reads one table. Closing takes a severity that means recovery, not merely one above the threshold that opened the event. Unknown is our own poll failing, so it holds the card open rather than reporting a recovery nobody saw. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/defaults.py | 7 ++ api/api/settings.py | 1 + api/polling/reconcile.py | 4 + api/polling/system_events.py | 166 +++++++++++++++++++++++++++ api/tests/test_system_events.py | 191 ++++++++++++++++++++++++++++++++ 5 files changed, 369 insertions(+) create mode 100644 api/polling/system_events.py create mode 100644 api/tests/test_system_events.py diff --git a/api/api/defaults.py b/api/api/defaults.py index b7d35e0..37315e1 100644 --- a/api/api/defaults.py +++ b/api/api/defaults.py @@ -119,6 +119,13 @@ class Throttle(StrEnum): # Long enough to read a week of failures, and no longer. POLL_RUN_RETENTION_DAYS = 30 +# How far back a provider's event may start and still claim one we +# opened. Providers back-date `starts_at` to when an incident really +# began, which is before our poll saw it. +EVENT_CLAIM_WINDOW = timedelta( + seconds=int(os.environ.get("EVENT_CLAIM_WINDOW_SECONDS", "3600")) +) + # Who the system writes as. RFC 2606 reserves the domain. No mail leaves # for it, and nobody can hold the address. SYSTEM_EMAIL = "system@statusboard.invalid" diff --git a/api/api/settings.py b/api/api/settings.py index b528ccd..341360b 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -13,6 +13,7 @@ APP_URL, DEFAULT_PAGE_SIZE, ENVIRONMENT, + EVENT_CLAIM_WINDOW, MAGIC_LINK_TTL, MAX_PAGE_SIZE, POLL_COOLDOWN_SECONDS, diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index 5ee19f9..24d79f5 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -5,6 +5,7 @@ from django.utils import timezone from catalog.models import ServiceComponent +from polling.system_events import reconcile_system_events from status.models import ComponentStatus, EventUpdate, ServiceEvent @@ -36,6 +37,9 @@ def apply_fetch(service, components, events, source, run=None): _archive_vanished(service, components, author) _write_statuses(components, rows, source, author, run) _upsert_events(service, events, rows, author, run) + # After the provider's events, so a component they explained does + # not also get one of ours in the same pass. + reconcile_system_events(service, author) def _signed(author, **fields): diff --git a/api/polling/system_events.py b/api/polling/system_events.py new file mode 100644 index 0000000..2fd823b --- /dev/null +++ b/api/polling/system_events.py @@ -0,0 +1,166 @@ +"""Events we open when a provider explains nothing. + +A provider can move a component to Degraded and never write an +incident. An event-only feed would hide that outage, and the closed +`ComponentStatus` span is exposed nowhere else. + +`ComponentStatus` stays the truth. An event here is a projection of it, +written by this module alone and rebuildable from nothing. That is what +keeps it from being a second answer to the same question. +""" + +from django.utils import timezone + +from status.choices import EventKind, EventSource, IncidentPhase, Severity +from status.models import ComponentStatus, EventUpdate, ServiceEvent + +# Worse than this and nothing is wrong. Severity 3 is our own poll +# failing to read their page. Severity 4 is a window a provider always +# announces. Neither is an outage of theirs to report. +SYSTEM_EVENT_MAX_SEVERITY = Severity.DEGRADED + +# Which severities end one of ours. Not the inverse of the rule above, +# on purpose. Unknown is absent because it is our own poll failing. A +# page we cannot read is no account of a recovery. Maintenance closes +# because a window a provider announces supersedes what we saw. +SYSTEM_EVENT_CLOSING_SEVERITIES = frozenset( + {Severity.OPERATIONAL, Severity.MAINTENANCE} +) + + +def reconcile_system_events(service, author): + """Open, extend and close the events this deployment owns.""" + for span in _bad_statuses(service): + _open_or_extend(span, author) + _close_recovered(service, author) + + +def _bad_statuses(service): + """Open spans bad enough to be somebody's outage.""" + return ComponentStatus.objects.filter( + component__service=service, + ended_at__isnull=True, + severity__lte=SYSTEM_EVENT_MAX_SEVERITY, + ).select_related("component") + + +def _explained(component): + """Whether a live provider event already accounts for this component. + + An incident that names no component covers its whole service. A + provider publishes one for an outage that reaches everything. One + outage is one card, so ours does not open beside it. + + A maintenance window naming nothing covers nothing. Planned work is + no account of an outage. Read as one, it hides every outage that + runs beside a window. + """ + live = ServiceEvent.objects.live().filter(detected_by=EventSource.PROVIDER) + if live.filter(affected_components=component).exists(): + return True + return live.filter( + service=component.service, + kind=EventKind.INCIDENT, + affected_components__isnull=True, + ).exists() + + +def _ours(component): + """The open event we opened for this component, if there is one.""" + return ( + ServiceEvent.objects.live() + .filter(affected_components=component, detected_by=EventSource.SYSTEM) + .first() + ) + + +def _open_or_extend(span, author): + component = span.component + if _explained(component): + return + event = _ours(component) + if event is None: + event = ServiceEvent.objects.create( + service=component.service, + external_id=None, + detected_by=EventSource.SYSTEM, + kind=EventKind.INCIDENT, + title=_title(component, span.severity), + phase=IncidentPhase.DETECTED, + starts_at=span.started_at, + created_by=author, + updated_by=author, + ) + event.affected_components.set([component]) + _record(event, span, author) + + +def _record(event, span, author): + """One update per severity span, keyed on when the span began. + + A poll that changes nothing runs this again. `get_or_create` on the + span's start is what stops a duplicate post per beat. + """ + EventUpdate.objects.get_or_create( + event=event, + posted_at=span.started_at, + defaults={ + "phase": event.phase, + "source": EventSource.SYSTEM, + "body": _title(span.component, span.severity), + "created_by": author, + "updated_by": author, + }, + ) + + +def _close_recovered(service, author): + """Close ours once the component it names is no longer bad.""" + open_events = ServiceEvent.objects.live().filter( + service=service, detected_by=EventSource.SYSTEM + ) + for event in open_events: + component = event.affected_components.first() + if component is None: + continue + current = ComponentStatus.objects.filter( + component=component, ended_at__isnull=True + ).first() + if _recovered(current): + _close(event, component, current, author) + + +def _recovered(current): + """Whether the open span says the outage is over. + + Closing needs a severity that means recovery, not merely one above + the threshold that opened the event. No open span closes it too. + """ + if current is None: + return True + return current.severity in SYSTEM_EVENT_CLOSING_SEVERITIES + + +def _close(event, component, current, author): + # A component with no open span has nothing wrong with it. The + # provider stopped listing it, and the card still has to close. + now = timezone.now() + severity = current.severity if current is not None else Severity.OPERATIONAL + EventUpdate.objects.create( + event=event, + posted_at=now, + phase=IncidentPhase.RESOLVED, + source=EventSource.SYSTEM, + body=_title(component, severity), + created_by=author, + updated_by=author, + ) + event.phase = IncidentPhase.RESOLVED + event.ends_at = now + event.updated_by = author + event.save(update_fields=["phase", "ends_at", "updated_by"]) + + +def _title(component, severity): + """What the card says before a provider gives it a better name.""" + return f"{component.name} {Severity(severity).label.lower()}" diff --git a/api/tests/test_system_events.py b/api/tests/test_system_events.py new file mode 100644 index 0000000..316c537 --- /dev/null +++ b/api/tests/test_system_events.py @@ -0,0 +1,191 @@ +import pytest +from django.contrib.auth import get_user_model +from django.utils import timezone + +from polling.system_events import reconcile_system_events +from status.choices import ( + EventKind, + EventSource, + IncidentPhase, + MaintenancePhase, + Severity, + StatusSource, +) +from status.models import ComponentStatus, ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory + +pytestmark = pytest.mark.django_db + + +def _status(component, severity): + ComponentStatus.objects.create( + component=component, + severity=severity, + source=StatusSource.PROVIDER, + started_at=timezone.now(), + ) + + +def _author(): + return get_user_model().objects.system() + + +def test_an_unexplained_outage_opens_an_event(): + # Without this the feed hides every outage a provider never wrote + # up, and the closed span is exposed nowhere else. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.DEGRADED) + + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.detected_by == EventSource.SYSTEM + assert event.external_id is None + assert event.phase == IncidentPhase.DETECTED + assert event.kind == EventKind.INCIDENT + assert list(event.affected_components.all()) == [component] + assert event.updates.count() == 1 + assert event.updates.first().source == EventSource.SYSTEM + + +def test_an_explained_outage_opens_nothing(): + # The provider already told the story. A second card for one + # outage is the thing this design exists to avoid. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + provider.affected_components.set([component]) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 0 + + +def test_a_service_wide_incident_explains_its_service(): + # One outage is one card. A provider incident naming no component + # covers every component of that service. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.DEGRADED) + ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Everything is down", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + ) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 0 + + +def test_a_service_wide_maintenance_window_explains_nothing(): + # A planned window is not an account of an outage. Read as one, a + # provider doing scheduled work hides every outage beside it. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.DEGRADED) + ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.MAINTENANCE, + title="Planned work", + phase=MaintenancePhase.IN_PROGRESS, + starts_at=timezone.now(), + ) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.filter(detected_by=EventSource.SYSTEM).count() == 1 + + +def test_unknown_opens_nothing(): + # Severity 3 is our own poll failing to read their page. Calling + # that their incident would report our fault as theirs. + service = ServiceFactory() + _status(ComponentFactory(service=service), Severity.UNKNOWN) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.count() == 0 + + +def test_every_transition_while_open_writes_an_update(): + # A card shows how the outage moved. One update at the start would + # say a major outage had been degraded the whole time. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.updates.count() == 2 + assert event.ends_at is None + + +def test_a_poll_that_changes_nothing_posts_nothing(): + # Without this every poll beat posts a duplicate update to an open + # card. The timeline then reads one line per beat, not per change. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + + reconcile_system_events(service, _author()) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.updates.count() == 1 + + +def test_unknown_holds_the_event_open(): + # Unknown is our poll failing to read their page. Closing on it + # tells somebody an outage ended when we stopped being able to see. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.UNKNOWN) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.DETECTED + assert event.ends_at is None + + +def test_recovery_closes_the_event(): + # An event that cannot close leaves a permanently red row on + # somebody's board. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + ComponentStatus.objects.filter(component=component, ended_at__isnull=True).update( + ended_at=timezone.now() + ) + _status(component, Severity.OPERATIONAL) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + assert event.phase == IncidentPhase.RESOLVED + assert event.ends_at is not None + assert event.updates.count() == 2 From 78c3bc90a9a4f8d6cc87a7e129ee008c45782b70 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 16:47:37 -0700 Subject: [PATCH 13/94] feat: a provider's post joins the event we already opened Twilio's dot goes red at 14:02 and they post at 14:14. Two rows would be two cards for one outage. Theirs is folded into ours, so the card gains updates rather than a duplicate, and detected_by survives. A claim merges their components onto ours, so the close rule now reads all of them. One recovery is no account of the others, and a card closed early is minted again on the next poll. Archiving also closes. A provider that stops publishing a component leaves an event nothing can ever recover, so a red row would sit on a board for good. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/polling/reconcile.py | 5 +- api/polling/system_events.py | 152 +++++++++++++++++--- api/tests/test_reconcile.py | 53 ++++++- api/tests/test_system_events.py | 240 +++++++++++++++++++++++++++++++- 4 files changed, 426 insertions(+), 24 deletions(-) diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index 24d79f5..ff96270 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -5,7 +5,7 @@ from django.utils import timezone from catalog.models import ServiceComponent -from polling.system_events import reconcile_system_events +from polling.system_events import claim, reconcile_system_events from status.models import ComponentStatus, EventUpdate, ServiceEvent @@ -230,3 +230,6 @@ def _upsert_events(service, events, rows, author, run=None): "updated_by": author, }, ) + # After the updates, so they move with the row if this event + # takes over one we opened. + claim(event, author) diff --git a/api/polling/system_events.py b/api/polling/system_events.py index 2fd823b..c7f2617 100644 --- a/api/polling/system_events.py +++ b/api/polling/system_events.py @@ -4,11 +4,16 @@ incident. An event-only feed would hide that outage, and the closed `ComponentStatus` span is exposed nowhere else. +A provider often posts after our poll saw the change. `claim` folds +their event into ours, so one outage stays one card. + `ComponentStatus` stays the truth. An event here is a projection of it, written by this module alone and rebuildable from nothing. That is what keeps it from being a second answer to the same question. """ +from django.conf import settings +from django.db import transaction from django.utils import timezone from status.choices import EventKind, EventSource, IncidentPhase, Severity @@ -35,10 +40,87 @@ def reconcile_system_events(service, author): _close_recovered(service, author) +@transaction.atomic +def claim(provider_event, author): + """Fold a provider's event into the one we opened for the same outage. + + A provider often posts minutes after our poll saw the change. Two + rows would be two cards for one outage. + + The provider's row is deleted and ours takes its id, so the fact + that we found it first survives. Deleting theirs rather than ours + is what keeps `detected_by` meaningful. + + Atomic on its own. A failure between the delete and the save loses + their row. Ours is then left without the id. The next poll writes + their event again, and moves duplicate updates onto ours. + """ + if provider_event.detected_by != EventSource.PROVIDER: + return False + # Only an incident claims. A claim copies their phase onto ours, + # and a maintenance phase on an incident is invalid. A planned + # window is also no account of the outage we found. + if provider_event.kind != EventKind.INCIDENT: + return False + ours = _claimable(provider_event) + if ours is None: + return False + ours.external_id = provider_event.external_id + ours.title = provider_event.title + ours.phase = provider_event.phase + ours.ends_at = provider_event.ends_at + ours.updated_by = author + # Their timeline moves first. A delete cascades to their updates, + # so the other order loses every post they made. + provider_event.updates.update(event=ours) + ours.affected_components.add(*provider_event.affected_components.all()) + # The unique key is the service and the provider id. Ours cannot + # hold their id while their row still does, so the save is last. + provider_event.delete() + ours.save(update_fields=["external_id", "title", "phase", "ends_at", "updated_by"]) + return True + + +def _claimable(provider_event): + """Our open event for the same outage, or nothing. + + A candidate is on the same service and names one of their + components. It began no later than their start plus the window. + The candidate whose start is nearest theirs wins when several match. + """ + components = list(provider_event.affected_components.all()) + if not components: + return None + # Read as "ours began no later than theirs plus the window". That + # is the same rule as "theirs began no earlier than ours less it". + floor = provider_event.starts_at + settings.EVENT_CLAIM_WINDOW + candidates = ( + ServiceEvent.objects.live() + .filter( + service=provider_event.service, + detected_by=EventSource.SYSTEM, + external_id__isnull=True, + affected_components__in=components, + starts_at__lte=floor, + ) + .distinct() + ) + return min( + candidates, + key=lambda e: abs(e.starts_at - provider_event.starts_at), + default=None, + ) + + def _bad_statuses(service): - """Open spans bad enough to be somebody's outage.""" + """Open spans bad enough to be somebody's outage. + + An archived component is left out. Its last span stays open for + good, so each poll would open a card that closes in the same pass. + """ return ComponentStatus.objects.filter( component__service=service, + component__is_archived=False, ended_at__isnull=True, severity__lte=SYSTEM_EVENT_MAX_SEVERITY, ).select_related("component") @@ -115,19 +197,56 @@ def _record(event, span, author): def _close_recovered(service, author): - """Close ours once the component it names is no longer bad.""" + """Close ours once every component it names is no longer bad.""" open_events = ServiceEvent.objects.live().filter( service=service, detected_by=EventSource.SYSTEM ) for event in open_events: - component = event.affected_components.first() - if component is None: - continue - current = ComponentStatus.objects.filter( - component=component, ended_at__isnull=True - ).first() - if _recovered(current): - _close(event, component, current, author) + closing = _closing(event) + if closing is not None: + _close(event, author, *closing) + + +def _closing(event): + """When and why an event ends, or nothing while a component is bad. + + A claim merges the provider's components onto ours, and they name + several. One recovery is no account of the others. + + The component that settles last decides both. Holding the card open + until then is the conservative error. Closing early leaves a + component down with nothing covering it. The next poll then opens a + second card for the same outage. + """ + now = timezone.now() + # The primary key sorts by creation, so this is a total order. The + # relation has none, and the pair below has to be repeatable. + components = list(event.affected_components.order_by("pk")) + if not components: + return None + settled = [] + for component in components: + ended = _settled(component, now) + if ended is None: + return None + settled.append(ended) + return max(settled, key=lambda pair: pair[0]) + + +def _settled(component, now): + """When and why one component stopped being bad, or nothing.""" + # A provider stopped publishing an archived component. Nothing can + # report its recovery, so the event would never close. + if component.is_archived: + return component.archived_at, f"{component.name} is no longer published" + current = ComponentStatus.objects.filter( + component=component, ended_at__isnull=True + ).first() + if not _recovered(current): + return None + # A component with no open span has nothing wrong with it. + severity = current.severity if current is not None else Severity.OPERATIONAL + return now, _title(component, severity) def _recovered(current): @@ -141,22 +260,19 @@ def _recovered(current): return current.severity in SYSTEM_EVENT_CLOSING_SEVERITIES -def _close(event, component, current, author): - # A component with no open span has nothing wrong with it. The - # provider stopped listing it, and the card still has to close. - now = timezone.now() - severity = current.severity if current is not None else Severity.OPERATIONAL +def _close(event, author, ends_at, body): + """Resolve one of ours, and post the update that says why.""" EventUpdate.objects.create( event=event, - posted_at=now, + posted_at=ends_at, phase=IncidentPhase.RESOLVED, source=EventSource.SYSTEM, - body=_title(component, severity), + body=body, created_by=author, updated_by=author, ) event.phase = IncidentPhase.RESOLVED - event.ends_at = now + event.ends_at = ends_at event.updated_by = author event.save(update_fields=["phase", "ends_at", "updated_by"]) diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index c410edf..912ff15 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -8,7 +8,13 @@ NormalisedUpdate, ) from polling.reconcile import apply_fetch, rebuild_ancestry, rebuild_search -from status.choices import EventKind, IncidentPhase, Severity, StatusSource +from status.choices import ( + EventKind, + EventSource, + IncidentPhase, + Severity, + StatusSource, +) from status.models import ComponentStatus, ServiceEvent from tests.factories import ComponentFactory, ServiceFactory @@ -157,6 +163,51 @@ def test_an_event_is_linked_to_the_components_it_names(): assert [c.external_id for c in event.affected_components.all()] == ["a"] +@pytest.mark.django_db +def test_a_later_provider_post_joins_the_event_the_poll_opened(): + # A poll saw the drop before the provider wrote it up. Without the + # claim the feed shows two cards for one outage. + service = ServiceFactory() + apply_fetch( + service, + [_component("a", severity=Severity.DEGRADED)], + [], + StatusSource.PROVIDER, + ) + ours = ServiceEvent.objects.get(service=service) + assert ours.detected_by == EventSource.SYSTEM + + apply_fetch( + service, + [_component("a", severity=Severity.DEGRADED)], + [ + NormalisedEvent( + external_id="inc-1", + kind=EventKind.INCIDENT, + title="Elevated SMS delivery failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + affected_external_ids=("a",), + updates=( + NormalisedUpdate( + phase=IncidentPhase.INVESTIGATING, + body="We are looking into it", + posted_at=timezone.now(), + ), + ), + ) + ], + StatusSource.PROVIDER, + ) + + event = ServiceEvent.objects.get(service=service) + assert event.pk == ours.pk + assert event.external_id == "inc-1" + # We found it first, and a claim never rewrites that. + assert event.detected_by == EventSource.SYSTEM + assert event.updates.count() == 2 + + @pytest.mark.django_db def test_a_run_stamps_what_it_wrote(): # A wrong or stale reading is otherwise untraceable: you can see what diff --git a/api/tests/test_system_events.py b/api/tests/test_system_events.py index 316c537..598ff66 100644 --- a/api/tests/test_system_events.py +++ b/api/tests/test_system_events.py @@ -1,8 +1,10 @@ +from datetime import timedelta + import pytest from django.contrib.auth import get_user_model from django.utils import timezone -from polling.system_events import reconcile_system_events +from polling.system_events import claim, reconcile_system_events from status.choices import ( EventKind, EventSource, @@ -11,18 +13,18 @@ Severity, StatusSource, ) -from status.models import ComponentStatus, ServiceEvent +from status.models import ComponentStatus, EventUpdate, ServiceEvent from tests.factories import ComponentFactory, ServiceFactory pytestmark = pytest.mark.django_db -def _status(component, severity): +def _status(component, severity, started_at=None): ComponentStatus.objects.create( component=component, severity=severity, source=StatusSource.PROVIDER, - started_at=timezone.now(), + started_at=started_at or timezone.now(), ) @@ -189,3 +191,233 @@ def test_recovery_closes_the_event(): assert event.phase == IncidentPhase.RESOLVED assert event.ends_at is not None assert event.updates.count() == 2 + + +def test_a_provider_event_claims_the_one_we_opened(): + # One outage is one card. Two cards for one outage is what the + # claim exists to prevent. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + voice = ComponentFactory(service=service, name="Voice") + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated SMS delivery failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(minutes=10), + ) + provider.affected_components.set([component, voice]) + EventUpdate.objects.create( + event=provider, + phase=IncidentPhase.INVESTIGATING, + body="We are looking into it", + posted_at=provider.starts_at, + ) + + assert claim(provider, _author()) is True + + ours.refresh_from_db() + assert ours.external_id == "abc" + assert ours.title == "Elevated SMS delivery failures" + assert ours.phase == IncidentPhase.INVESTIGATING + # Ours records who found it, and a claim never rewrites that. + assert ours.detected_by == EventSource.SYSTEM + assert not ServiceEvent.objects.filter(pk=provider.pk).exists() + # Their post moves before their row goes. Deleting first would + # cascade the timeline away. + assert ours.updates.count() == 2 + assert {u.source for u in ours.updates.all()} == { + EventSource.SYSTEM, + EventSource.PROVIDER, + } + # Their event named a component ours did not. Losing it would drop + # that component from the card. + assert set(ours.affected_components.all()) == {component, voice} + + +def test_a_stale_provider_event_claims_nothing(): + # An incident that began a day before ours is a different outage. + # Merging them would put one card's updates on another's timeline. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Yesterday", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at - timedelta(days=1), + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is False + assert ServiceEvent.objects.count() == 2 + + +def test_a_maintenance_window_claims_nothing(): + # A claim copies their phase onto ours. A maintenance phase on an + # incident is a combination no provider publishes. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.MAINTENANCE, + title="Planned work", + phase=MaintenancePhase.IN_PROGRESS, + starts_at=ours.starts_at, + ) + provider.affected_components.set([component]) + + assert claim(provider, _author()) is False + ours.refresh_from_db() + assert ours.phase == IncidentPhase.DETECTED + + +def test_one_of_ours_claims_nothing(): + # A claim runs on a provider's row. Ours would match itself, take + # its own id and delete the row it was folding into. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + assert claim(ours, _author()) is False + assert ServiceEvent.objects.count() == 1 + + +def test_a_claimed_event_is_not_claimed_twice(): + # The next poll finds ours by the id it took, so `update_or_create` + # updates it in place. A second claim would delete the row it just + # matched. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.DEGRADED) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at, + ) + provider.affected_components.set([component]) + claim(provider, _author()) + + ours.refresh_from_db() + assert claim(ours, _author()) is False + assert ServiceEvent.objects.count() == 1 + + +def test_archiving_a_component_closes_our_open_event(): + # We can no longer watch it recover, so the event could never + # close. A red row would stay on a board for good. + service = ServiceFactory() + component = ComponentFactory(service=service, name="SMS") + _status(component, Severity.MAJOR_OUTAGE) + reconcile_system_events(service, _author()) + + component.is_archived = True + component.save(update_fields=["is_archived"]) + reconcile_system_events(service, _author()) + + event = ServiceEvent.objects.get(service=service) + component.refresh_from_db() + assert event.phase == IncidentPhase.RESOLVED + # The end is when the provider dropped it, not when we noticed. + # Stamping now would report an outage that ran until this poll. + assert event.ends_at == component.archived_at + assert event.updates.latest("posted_at").body == "SMS is no longer published" + + +def test_an_archived_component_opens_nothing(): + # Opening one we can never watch recover writes a card that closes + # in the same pass, once per poll. + service = ServiceFactory() + component = ComponentFactory(service=service) + _status(component, Severity.MAJOR_OUTAGE) + component.is_archived = True + component.save(update_fields=["is_archived"]) + + reconcile_system_events(service, _author()) + + assert ServiceEvent.objects.count() == 0 + + +def test_the_nearest_start_wins_when_several_could_be_claimed(): + # Their post explains the outage that began beside it. The further + # event is another outage, and their updates would land on it. + service = ServiceFactory() + early = ComponentFactory(service=service, name="SMS") + late = ComponentFactory(service=service, name="Voice") + now = timezone.now() + _status(early, Severity.DEGRADED, started_at=now - timedelta(minutes=30)) + _status(late, Severity.DEGRADED, started_at=now - timedelta(minutes=5)) + reconcile_system_events(service, _author()) + further = ServiceEvent.objects.get(affected_components=early) + nearer = ServiceEvent.objects.get(affected_components=late) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=now - timedelta(minutes=6), + ) + provider.affected_components.set([early, late]) + + assert claim(provider, _author()) is True + + nearer.refresh_from_db() + further.refresh_from_db() + assert nearer.external_id == "abc" + assert further.external_id is None + + +def test_a_claimed_event_stays_open_while_one_component_is_bad(): + # A claim merges their components onto ours. One recovery is no + # account of the others, and a closed card is minted again next poll. + service = ServiceFactory() + sms = ComponentFactory(service=service, name="SMS") + voice = ComponentFactory(service=service, name="Voice") + _status(sms, Severity.MAJOR_OUTAGE) + _status(voice, Severity.OPERATIONAL) + reconcile_system_events(service, _author()) + ours = ServiceEvent.objects.get(service=service) + + provider = ServiceEvent.objects.create( + service=service, + external_id="abc", + kind=EventKind.INCIDENT, + title="Elevated failures", + phase=IncidentPhase.INVESTIGATING, + starts_at=ours.starts_at, + ) + provider.affected_components.set([sms, voice]) + assert claim(provider, _author()) is True + + reconcile_system_events(service, _author()) + reconcile_system_events(service, _author()) + + ours.refresh_from_db() + assert ours.phase == IncidentPhase.INVESTIGATING + assert ours.ends_at is None + assert ServiceEvent.objects.count() == 1 From 68a481a3420703f9fa83a3bf3b44388766c4c8ea Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 17:16:47 -0700 Subject: [PATCH 14/94] feat: one event feed for Home, a service and a component Three screens showed the same list at different scopes. The phase filter draws its line from CLOSED_PHASES, so no client carries a copy of which phases are terminal. Moved the board fixture into conftest.py and added a shared jwt_client helper, because DEFAULT_AUTHENTICATION_CLASSES is JWT only and force_login cannot authenticate a DRF view here. test_board_api.py's client fixture now delegates to it instead of keeping a second copy. Added a fields probe for /events/ to test_contract.py, so ?fields= is proven to prune the feed the same way every other GET does. get_queryset no longer parses ?dashboard= itself before EventFilter does. A malformed value used to reach get_object_or_404 as a raw string and raise an uncaught ValueError, a 500 that common/errors.py exists to prevent. It now only decides ownership once EventFilter's own UUIDFilter has confirmed the value parses, and lets that filter reject a bad one in the same shape any other malformed filter value gets. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/settings.py | 1 + api/api/urls.py | 1 + api/status/filters.py | 42 ++++++++++ api/status/urls.py | 7 ++ api/status/views.py | 60 ++++++++++++++ api/tests/conftest.py | 29 +++++++ api/tests/test_board_api.py | 19 ++--- api/tests/test_contract.py | 17 +++- api/tests/test_event_api.py | 159 ++++++++++++++++++++++++++++++++++++ docs/api/openapi.yaml | 36 ++++++++ 10 files changed, 355 insertions(+), 16 deletions(-) create mode 100644 api/status/filters.py create mode 100644 api/status/urls.py create mode 100644 api/status/views.py create mode 100644 api/tests/test_event_api.py diff --git a/api/api/settings.py b/api/api/settings.py index 341360b..1677ac1 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -386,6 +386,7 @@ {"name": "auth"}, {"name": "me"}, {"name": "catalog"}, + {"name": "status"}, {"name": "board"}, ], } diff --git a/api/api/urls.py b/api/api/urls.py index 3057810..6b6d751 100644 --- a/api/api/urls.py +++ b/api/api/urls.py @@ -17,4 +17,5 @@ path("", include("authentication.urls")), path("", include("catalog.urls")), path("", include("dashboards.urls")), + path("", include("status.urls")), ] diff --git a/api/status/filters.py b/api/status/filters.py new file mode 100644 index 0000000..8ec387e --- /dev/null +++ b/api/status/filters.py @@ -0,0 +1,42 @@ +"""What a caller may narrow the event feed by. + +Beside the views, and apart from them. A filter is the query contract, +and it is read on its own more often than the view around it. +""" + +from django_filters import rest_framework as filters + +from status.choices import CLOSED_PHASES +from status.models import ServiceEvent + +PHASE_STATES = [("open", "Open"), ("closed", "Closed")] + + +class EventFilter(filters.FilterSet): + # Declared, not generated. `CLOSED_PHASES` draws the line between + # an open phase and a closed one. A client restating it is a + # second copy of one rule. + phase = filters.ChoiceFilter(choices=PHASE_STATES, method="filter_phase") + service = filters.CharFilter(field_name="service__slug") + component = filters.UUIDFilter(field_name="affected_components__id") + dashboard = filters.UUIDFilter(method="filter_dashboard") + + class Meta: + model = ServiceEvent + fields = {"kind": ["exact"]} + + def filter_phase(self, queryset, name, value): + if value == "closed": + return queryset.filter(phase__in=CLOSED_PHASES) + return queryset.exclude(phase__in=CLOSED_PHASES) + + def filter_dashboard(self, queryset, name, value): + """Everything posted across the services on one board. + + The board's rows are components, and an event names components. + A user reaches only their own board: the view checks the owner + before this runs. + """ + return queryset.filter( + affected_components__tracked_by__dashboard_id=value + ).distinct() diff --git a/api/status/urls.py b/api/status/urls.py new file mode 100644 index 0000000..ffd2a32 --- /dev/null +++ b/api/status/urls.py @@ -0,0 +1,7 @@ +from django.urls import path + +from status.views import EventListView + +urlpatterns = [ + path("events/", EventListView.as_view(), name="event-list"), +] diff --git a/api/status/views.py b/api/status/views.py new file mode 100644 index 0000000..117adc1 --- /dev/null +++ b/api/status/views.py @@ -0,0 +1,60 @@ +"""Every read of an event, in one place. + +Home, a service and a component show the same feed at different scopes. +The screens call it Updates. This names the model, and `/meta/` +publishes the labels. +""" + +import uuid + +from django.shortcuts import get_object_or_404 +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import generics +from rest_framework.exceptions import NotAuthenticated +from rest_framework.permissions import AllowAny + +from common.aggregates import EventAggregateSet +from common.filters import FieldsBackend +from common.ordering import MappedOrderingFilter +from dashboards.models import Dashboard +from status.filters import EventFilter +from status.models import ServiceEvent +from status.serializers import ServiceEventSerializer + + +class EventListView(generics.ListAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventSerializer + aggregate_set = EventAggregateSet + filterset_class = EventFilter + filter_backends = [DjangoFilterBackend, MappedOrderingFilter, FieldsBackend] + ordering_fields = ["starts_at", "ends_at"] + ordering_map = {} + ordering = ["-starts_at"] + # get_queryset reads request.query_params, which schema generation + # has not set. This names the model without running it. + queryset = ServiceEvent.objects.none() + + def get_queryset(self): + queryset = ServiceEvent.objects.select_related("service").prefetch_related( + "updates" + ) + board = self.request.query_params.get("dashboard") + if not board: + return queryset + try: + uuid.UUID(board) + except ValueError: + # Not a UUID. `EventFilter`'s own `dashboard` field rejects + # it next, in the shape any other malformed filter value + # gets. This method only owns whether the board is yours. + return queryset + # An anonymous caller cannot own a board. Refuse before + # comparing against AnonymousUser, which get_object_or_404 + # cannot do. + if not self.request.user.is_authenticated: + raise NotAuthenticated + # 404 rather than 403: someone else's board id should not + # be confirmable. The filter runs after this. + get_object_or_404(Dashboard, id=board, owner=self.request.user) + return queryset diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 1bee08b..5cf0f90 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -1,4 +1,33 @@ import pytest +from rest_framework.test import APIClient +from rest_framework_simplejwt.tokens import RefreshToken + +from authentication.models import User + + +@pytest.fixture +def board(db): + user = User.objects.create(email="a@b.com") + return user.default_dashboard + + +def jwt_client(user): + """An `APIClient` bearing a fresh access token for this user. + + `DEFAULT_AUTHENTICATION_CLASSES` is JWT only. `force_login` leaves + a DRF view answering 401, because there is no session backend. + """ + api = APIClient() + api.credentials( + HTTP_AUTHORIZATION=f"Bearer {RefreshToken.for_user(user).access_token}" + ) + return api + + +@pytest.fixture +def authenticated_client(board): + """A board owner's own client. Most tests want the anonymous one.""" + return jwt_client(board.owner) @pytest.fixture(autouse=True) diff --git a/api/tests/test_board_api.py b/api/tests/test_board_api.py index 8b3c9a7..4659bf2 100644 --- a/api/tests/test_board_api.py +++ b/api/tests/test_board_api.py @@ -22,20 +22,11 @@ @pytest.fixture -def board(db): - user = User.objects.create(email="a@b.com") - return user.default_dashboard - - -@pytest.fixture -def client(board): - from rest_framework_simplejwt.tokens import RefreshToken - - api = APIClient() - api.credentials( - HTTP_AUTHORIZATION=f"Bearer {RefreshToken.for_user(board.owner).access_token}" - ) - return api +def client(authenticated_client): + # Every test in this module reads or writes one board as its + # owner. The plain `client` fixture stays anonymous everywhere + # else, so this override is local to this module. + return authenticated_client def _track(board, severity=Severity.OPERATIONAL, slug=None): diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py index f1f46e3..62d9189 100644 --- a/api/tests/test_contract.py +++ b/api/tests/test_contract.py @@ -114,6 +114,19 @@ def _service_events_probe(): return APIClient(), url, "id" +def _event_list_probe(): + service = ServiceFactory() + ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + return APIClient(), reverse("event-list"), "id" + + def _component_list_probe(): service = ServiceFactory() ComponentFactory(service=service, is_overall=True) @@ -138,8 +151,7 @@ def _board_components_probe(): # One builder per documented GET path: a client, a live URL, and a field -# to keep. `/events/` and its children are not here. They are not in -# the contract yet, so this test does not cover them until they land. +# to keep. FIELDS_PROBES = { "/meta/": _meta_probe, "/me/": _me_probe, @@ -150,6 +162,7 @@ def _board_components_probe(): "/catalog/components/": _component_list_probe, "/catalog/components/{uuid}/": _component_detail_probe, "/dashboards/{uuid}/components/": _board_components_probe, + "/events/": _event_list_probe, } diff --git a/api/tests/test_event_api.py b/api/tests/test_event_api.py new file mode 100644 index 0000000..61a0da1 --- /dev/null +++ b/api/tests/test_event_api.py @@ -0,0 +1,159 @@ +import pytest +from django.urls import reverse +from django.utils import timezone + +from authentication.models import User +from status.choices import EventKind, IncidentPhase, MaintenancePhase +from status.models import ServiceEvent +from tests.factories import ComponentFactory, ServiceFactory, track + +pytestmark = pytest.mark.django_db + + +def _event(service, component, **kwargs): + fields = { + "kind": EventKind.INCIDENT, + "title": "Something broke", + "phase": IncidentPhase.INVESTIGATING, + "starts_at": timezone.now(), + } + fields.update(kwargs) + event = ServiceEvent.objects.create(service=service, **fields) + event.affected_components.set([component]) + return event + + +def test_the_feed_lists_events_newest_first(client): + # A feed is read from the top. Oldest first would open on an + # incident from last month. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Older") + new = _event(service, component, external_id="2", title="Newer") + ServiceEvent.objects.filter(pk=new.pk).update( + starts_at=timezone.now() + timezone.timedelta(minutes=5) + ) + + results = client.get(reverse("event-list")).json()["results"] + assert [r["title"] for r in results] == ["Newer", "Older"] + + +def test_service_narrows_the_feed(client): + # A service's Updates tab. Anything from another service on it + # would be answering a question nobody asked on that page. + first = ServiceFactory() + second = ServiceFactory() + _event(first, ComponentFactory(service=first), external_id="1", title="Mine") + _event(second, ComponentFactory(service=second), external_id="2", title="Theirs") + + results = client.get(reverse("event-list"), {"service": first.slug}).json()[ + "results" + ] + assert [r["title"] for r in results] == ["Mine"] + + +def test_component_narrows_the_feed(client): + # A component's Updates tab reads the same collection one level + # down, so it cannot be a different endpoint. + service = ServiceFactory() + wanted = ComponentFactory(service=service) + other = ComponentFactory(service=service) + _event(service, wanted, external_id="1", title="Wanted") + _event(service, other, external_id="2", title="Other") + + results = client.get(reverse("event-list"), {"component": str(wanted.id)}).json()[ + "results" + ] + assert [r["title"] for r in results] == ["Wanted"] + + +def test_phase_open_and_closed_draw_one_line(client): + # `CLOSED_PHASES` lives in status/choices.py. A client restating + # which phases are terminal is a second copy that can drift. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Open") + _event( + service, + component, + external_id="2", + title="Done", + phase=IncidentPhase.RESOLVED, + ) + _event( + service, + component, + external_id="3", + title="Finished", + kind=EventKind.MAINTENANCE, + phase=MaintenancePhase.COMPLETED, + ) + + open_titles = client.get(reverse("event-list"), {"phase": "open"}).json()["results"] + closed = client.get(reverse("event-list"), {"phase": "closed"}).json()["results"] + assert [r["title"] for r in open_titles] == ["Open"] + assert {r["title"] for r in closed} == {"Done", "Finished"} + + +def test_dashboard_narrows_to_what_you_track(authenticated_client, board): + # Home's Updates tab. Everything posted across the services on + # your board, and nothing else. + # + # `DEFAULT_AUTHENTICATION_CLASSES` is JWT only, so a plain + # `force_login` leaves this view answering 401. + # `authenticated_client` bears board.owner's own access token. + tracked = ComponentFactory() + untracked = ComponentFactory() + track(tracked, user=board.owner) + _event(tracked.service, tracked, external_id="1", title="Yours") + _event(untracked.service, untracked, external_id="2", title="Not yours") + + results = authenticated_client.get( + reverse("event-list"), {"dashboard": str(board.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Yours"] + + +def test_an_anonymous_caller_cannot_narrow_to_a_dashboard(client, board): + # `get_object_or_404(..., owner=request.user)` would compare against + # `AnonymousUser` and raise `TypeError` instead of answering cleanly. + response = client.get(reverse("event-list"), {"dashboard": str(board.id)}) + assert response.status_code == 401 + + +def test_a_dashboard_that_is_not_yours_is_not_readable(authenticated_client): + # 404, not 403: a 403 would let anyone holding a valid token + # confirm whether a board id exists. + other = User.objects.create(email="someone-else@example.test") + response = authenticated_client.get( + reverse("event-list"), {"dashboard": str(other.default_dashboard.id)} + ) + assert response.status_code == 404 + + +def test_a_malformed_dashboard_id_is_a_400_not_a_500(authenticated_client): + # `EventFilter`'s `dashboard` field is a `UUIDFilter`. A bad value + # reaches django-filter and comes back like any other malformed + # filter value. It must not surface as an unhandled ValueError. + response = authenticated_client.get( + reverse("event-list"), {"dashboard": "not-a-uuid"} + ) + assert response.status_code == 400 + + +def test_the_dashboard_filter_does_not_repeat_an_event(authenticated_client, board): + # One event naming two components tracked on the same board is one + # outage. Without .distinct() the M2M join returns one row per + # matching component, and the feed shows it twice. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service) + track(first, user=board.owner) + track(second, user=board.owner) + event = _event(service, first, external_id="1", title="Shared") + event.affected_components.add(second) + + results = authenticated_client.get( + reverse("event-list"), {"dashboard": str(board.id)} + ).json()["results"] + assert [r["title"] for r in results] == ["Shared"] diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 0a7fb3f..97df7da 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -105,6 +105,8 @@ tags: - name: me - name: catalog description: Public. Services, their components, and incidents. + - name: status + description: Public. The event feed, one collection for Home, a service and a component. - name: board paths: @@ -380,6 +382,40 @@ paths: aggregates: { $ref: '#/components/schemas/EventAggregates' } results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } + /events/: + get: + tags: [status] + summary: The event feed + description: | + One feed, three scopes. Home with `dashboard`, a service with `service`, a component + with `component`. + + `phase` takes `open` or `closed`. The line between them is `CLOSED_PHASES` in the + code, so a client never restates which phases are terminal. + security: [{}, { bearer: [] }] + parameters: + - { name: dashboard, in: query, schema: { type: string, format: uuid }, description: Requires auth, and must be your own } + - { name: service, in: query, schema: { type: string }, description: Service slug } + - { name: component, in: query, schema: { type: string, format: uuid } } + - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] } } + - { name: phase, in: query, schema: { type: string, enum: [open, closed] } } + - { name: ordering, in: query, schema: { type: string, enum: [-starts_at, starts_at, ends_at, -ends_at], default: -starts_at } } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/EventAggregates' } + results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } + /dashboards/{uuid}/components/: get: tags: [board] From 2051da2d5ceb3e847c7a0ebbb299688ac1fe07d3 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 17:47:29 -0700 Subject: [PATCH 15/94] feat: an event, and its log as its own list The header draws both tab badges before either tab asks for anything, so the detail carries both counts. The log is paged on its own: a provider writes as many updates as they like. ServiceEventSerializer drops the nested `updates` array and gains `detected_by` and a nested `service` ref, so a row from `/events/` can name which service it belongs to and Home's Updates tab can label its rows. The service ref import is lazy: catalog.serializers already imports status.serializers, so a module-level import back would cycle. EventUpdate.source and ServiceEvent.detected_by share one choice set, which the schema generator was naming twice. ENUM_NAME_OVERRIDES gives it one name. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/settings.py | 5 +++ api/catalog/views.py | 2 +- api/common/pagination.py | 11 ++++++ api/status/serializers.py | 61 ++++++++++++++++++++++++++++- api/status/urls.py | 8 +++- api/status/views.py | 47 +++++++++++++++++++--- api/tests/test_catalog_api.py | 3 +- api/tests/test_contract.py | 38 +++++++++++++++++- api/tests/test_event_api.py | 74 ++++++++++++++++++++++++++++++++++- docs/api/openapi.yaml | 70 ++++++++++++++++++++++++++++----- docs/check_docs.py | 5 ++- 11 files changed, 300 insertions(+), 24 deletions(-) diff --git a/api/api/settings.py b/api/api/settings.py index 1677ac1..0b40725 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -381,6 +381,11 @@ "VERSION": "1.0.0", "SERVE_INCLUDE_SCHEMA": False, "COMPONENT_SPLIT_REQUEST": True, + # EventUpdate.source and ServiceEvent.detected_by share one choice + # set. Without this, the generator names the same enum twice. + "ENUM_NAME_OVERRIDES": { + "EventSourceEnum": "status.choices.EventSource", + }, "TAGS": [ {"name": "meta"}, {"name": "auth"}, diff --git a/api/catalog/views.py b/api/catalog/views.py index 853bfc0..07afa5f 100644 --- a/api/catalog/views.py +++ b/api/catalog/views.py @@ -131,7 +131,7 @@ def components(self, request, slug=None): ) def events(self, request, slug=None): return self._page( - ServiceEvent.objects.filter(service__slug=slug).prefetch_related("updates") + ServiceEvent.objects.filter(service__slug=slug).select_related("service") ) diff --git a/api/common/pagination.py b/api/common/pagination.py index acd116f..aa76b78 100644 --- a/api/common/pagination.py +++ b/api/common/pagination.py @@ -32,3 +32,14 @@ def get_paginated_response(self, data): "results": data, } ) + + +class TimelinePagination(EnvelopePagination): + """One event's log, oldest first. + + `posted_at` is not unique on its own, so `created_at` breaks a tie + between two updates posted in the same second. + """ + + ordering = "posted_at" + tiebreak = "created_at" diff --git a/api/status/serializers.py b/api/status/serializers.py index 6def9cf..eb25847 100644 --- a/api/status/serializers.py +++ b/api/status/serializers.py @@ -5,6 +5,14 @@ from status.models import ComponentStatus, EventUpdate, ServiceEvent +def _service_ref_serializer(): + # Imported lazily. catalog imports status, so a module-level import + # here would be a cycle. + from catalog.serializers import ServiceRefSerializer + + return ServiceRefSerializer + + class StatusSerializer(FieldsMixin, serializers.ModelSerializer): last_refreshed_at = serializers.SerializerMethodField() @@ -36,8 +44,57 @@ class Meta: class ServiceEventSerializer(FieldsMixin, serializers.ModelSerializer): - updates = EventUpdateSerializer(many=True, read_only=True) + """One row of the feed. The log is `/events/{uuid}/updates/`.""" class Meta: model = ServiceEvent - fields = ["id", "kind", "title", "phase", "starts_at", "ends_at", "updates"] + fields = [ + "id", + "kind", + "title", + "phase", + "starts_at", + "ends_at", + "detected_by", + "service", + ] + + def get_fields(self): + # `catalog.serializers` imports this module for `EventRefSerializer`, + # so the reverse import has to wait until both have finished loading. + fields = super().get_fields() + fields["service"] = _service_ref_serializer()(read_only=True) + return fields + + +class ServiceEventDetailSerializer(ServiceEventSerializer): + """The event screen's header and its About tab. + + Both counts are here because the header draws both tab badges + before either tab has made a request. + """ + + update_count = serializers.SerializerMethodField() + affected_count = serializers.SerializerMethodField() + last_update_at = serializers.SerializerMethodField() + + class Meta(ServiceEventSerializer.Meta): + fields = [ + *ServiceEventSerializer.Meta.fields, + "update_count", + "affected_count", + "last_update_at", + ] + + @extend_schema_field(serializers.IntegerField()) + def get_update_count(self, event): + return event.updates.count() + + @extend_schema_field(serializers.IntegerField()) + def get_affected_count(self, event): + return event.affected_components.count() + + @extend_schema_field(serializers.DateTimeField(allow_null=True)) + def get_last_update_at(self, event): + newest = event.updates.order_by("-posted_at").first() + return newest.posted_at if newest else None diff --git a/api/status/urls.py b/api/status/urls.py index ffd2a32..05495a0 100644 --- a/api/status/urls.py +++ b/api/status/urls.py @@ -1,7 +1,13 @@ from django.urls import path -from status.views import EventListView +from status.views import EventDetailView, EventListView, EventUpdateListView urlpatterns = [ path("events/", EventListView.as_view(), name="event-list"), + path("events//", EventDetailView.as_view(), name="event-detail"), + path( + "events//updates/", + EventUpdateListView.as_view(), + name="event-updates", + ), ] diff --git a/api/status/views.py b/api/status/views.py index 117adc1..ec80b6a 100644 --- a/api/status/views.py +++ b/api/status/views.py @@ -16,10 +16,15 @@ from common.aggregates import EventAggregateSet from common.filters import FieldsBackend from common.ordering import MappedOrderingFilter +from common.pagination import TimelinePagination from dashboards.models import Dashboard from status.filters import EventFilter -from status.models import ServiceEvent -from status.serializers import ServiceEventSerializer +from status.models import EventUpdate, ServiceEvent +from status.serializers import ( + EventUpdateSerializer, + ServiceEventDetailSerializer, + ServiceEventSerializer, +) class EventListView(generics.ListAPIView): @@ -36,9 +41,7 @@ class EventListView(generics.ListAPIView): queryset = ServiceEvent.objects.none() def get_queryset(self): - queryset = ServiceEvent.objects.select_related("service").prefetch_related( - "updates" - ) + queryset = ServiceEvent.objects.select_related("service") board = self.request.query_params.get("dashboard") if not board: return queryset @@ -58,3 +61,37 @@ def get_queryset(self): # be confirmable. The filter runs after this. get_object_or_404(Dashboard, id=board, owner=self.request.user) return queryset + + +class EventDetailView(generics.RetrieveAPIView): + permission_classes = [AllowAny] + serializer_class = ServiceEventDetailSerializer + filter_backends = [FieldsBackend] + lookup_field = "pk" + # The contract names the path variable `uuid`. DRF renames a `pk` + # one to `id`, which would document a path nothing serves. + lookup_url_kwarg = "uuid" + queryset = ServiceEvent.objects.select_related("service") + + +class EventUpdateListView(generics.ListAPIView): + """One event's log, oldest first. + + The feed is newest first because it is a feed. This is a + narrative, and a narrative is read forwards. + """ + + permission_classes = [AllowAny] + serializer_class = EventUpdateSerializer + filter_backends = [FieldsBackend] + pagination_class = TimelinePagination + # get_queryset reads self.kwargs, which schema generation has not + # set. This names the model without running it. + queryset = EventUpdate.objects.none() + + def get_queryset(self): + event = get_object_or_404(ServiceEvent, pk=self.kwargs["uuid"]) + # `TimelinePagination.ordering` is what actually orders the + # response: CursorPagination always re-sorts by it. This is + # the correct order regardless, in case pagination is ever off. + return event.updates.order_by("posted_at") diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index 153ec03..4b4b56f 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -222,7 +222,8 @@ def test_an_event_carries_no_component_ids(): "phase", "starts_at", "ends_at", - "updates", + "detected_by", + "service", } diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py index 62d9189..2a41c0d 100644 --- a/api/tests/test_contract.py +++ b/api/tests/test_contract.py @@ -12,7 +12,7 @@ from authentication.models import User from dashboards.models import DashboardItem from status.choices import EventKind, IncidentPhase -from status.models import ServiceEvent +from status.models import EventUpdate, ServiceEvent from tests.factories import ComponentFactory, ServiceFactory, StatusPageFactory CONTRACT = Path(__file__).resolve().parents[2] / "docs" / "api" / "openapi.yaml" @@ -127,6 +127,40 @@ def _event_list_probe(): return APIClient(), reverse("event-list"), "id" +def _event_detail_probe(): + service = ServiceFactory() + event = ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + url = reverse("event-detail", kwargs={"uuid": event.id}) + return APIClient(), url, "id" + + +def _event_updates_probe(): + service = ServiceFactory() + event = ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.DETECTED, + body="x", + posted_at=timezone.now(), + ) + url = reverse("event-updates", kwargs={"uuid": event.id}) + return APIClient(), url, "phase" + + def _component_list_probe(): service = ServiceFactory() ComponentFactory(service=service, is_overall=True) @@ -163,6 +197,8 @@ def _board_components_probe(): "/catalog/components/{uuid}/": _component_detail_probe, "/dashboards/{uuid}/components/": _board_components_probe, "/events/": _event_list_probe, + "/events/{uuid}/": _event_detail_probe, + "/events/{uuid}/updates/": _event_updates_probe, } diff --git a/api/tests/test_event_api.py b/api/tests/test_event_api.py index 61a0da1..ddc2aca 100644 --- a/api/tests/test_event_api.py +++ b/api/tests/test_event_api.py @@ -3,8 +3,8 @@ from django.utils import timezone from authentication.models import User -from status.choices import EventKind, IncidentPhase, MaintenancePhase -from status.models import ServiceEvent +from status.choices import EventKind, EventSource, IncidentPhase, MaintenancePhase +from status.models import EventUpdate, ServiceEvent from tests.factories import ComponentFactory, ServiceFactory, track pytestmark = pytest.mark.django_db @@ -38,6 +38,16 @@ def test_the_feed_lists_events_newest_first(client): assert [r["title"] for r in results] == ["Newer", "Older"] +def test_a_feed_row_names_its_service(client): + # /events/?dashboard= spans every service on a board. A row cannot + # label itself without the service it belongs to. + service = ServiceFactory(slug="pagerduty") + _event(service, ComponentFactory(service=service), external_id="1") + + results = client.get(reverse("event-list")).json()["results"] + assert results[0]["service"]["slug"] == "pagerduty" + + def test_service_narrows_the_feed(client): # A service's Updates tab. Anything from another service on it # would be answering a question nobody asked on that page. @@ -157,3 +167,63 @@ def test_the_dashboard_filter_does_not_repeat_an_event(authenticated_client, boa reverse("event-list"), {"dashboard": str(board.id)} ).json()["results"] assert [r["title"] for r in results] == ["Shared"] + + +def test_the_detail_carries_both_tab_counts(client): + # The header draws `Timeline 3` and `Affects 2` before either tab + # is opened, so neither can wait for that tab's request. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service) + event = _event(service, first, external_id="1") + event.affected_components.add(second) + for minute in range(3): + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body=f"update {minute}", + posted_at=timezone.now(), + ) + + body = client.get(reverse("event-detail", args=[event.id])).json() + assert body["update_count"] == 3 + assert body["affected_count"] == 2 + + +def test_the_timeline_is_its_own_paged_list(client): + # A provider's log has no ceiling, so it cannot ride on the detail. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Cause found", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) + + body = client.get(reverse("event-updates", args=[event.id])).json() + assert body["aggregates"]["total"] == 1 + assert body["results"][0]["source"] == "provider" + + +def test_the_timeline_is_oldest_first(client): + # A story is read forwards. The feed is newest first because it is + # a feed; one event's log is a narrative. + service = ServiceFactory() + event = _event(service, ComponentFactory(service=service), external_id="1") + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.IDENTIFIED, + body="Second", + posted_at=timezone.now() + timezone.timedelta(minutes=1), + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body="First", + posted_at=timezone.now(), + ) + + results = client.get(reverse("event-updates", args=[event.id])).json()["results"] + assert [r["body"] for r in results] == ["First", "Second"] diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 97df7da..a1c3347 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -41,7 +41,7 @@ info: Do not derive one from the other. - ## One schema, one model + ## One schema, one model, usually Every schema maps to one Django model, and every nested key is a real relation a `ModelSerializer` can traverse. Nest a related model under a key. Do not flatten it into the @@ -54,8 +54,8 @@ info: | `Service.overall_component` | the `ServiceComponent` where `is_overall` | | `Component.status` | the `ComponentStatus` row where `ended_at` is null | | `Component.active_incident` | newest unresolved `ServiceEvent` with `kind=incident`, across the reverse M2M | - | `ServiceEvent.updates` | reverse FK from `EventUpdate` | | `Component.service` | `ForeignKey` | + | `ServiceEvent.service` | `ForeignKey` | `status` is its own object because `ComponentStatus` is its own table. A `ModelSerializer` with a nested serializer produces these shapes directly. @@ -63,12 +63,11 @@ info: The rule applies transitively. A `Service` nests its overall component. It does not copy that component's status. - **One model, one schema.** There is no list variant and detail variant of the same thing. - A second schema exists only to withhold fields, and every field here is either a column on the - row or a `select_related` join — nothing a variant would save is expensive. `?fields=` lets a - client ask for a smaller shape, and unlike a schema variant the client chooses. + **A second schema never exists to withhold fields.** `?fields=` does that, and the client + chooses. One exists only to layer a value too expensive to compute for every row in a list. - `allOf` therefore appears only where two schemas cover two models. + `allOf` appears where two schemas cover two models. It also appears where a detail schema + layers that kind of value onto its own list row. Query parameters use the ORM path. django-filter generates them from `filterset_fields` and `ordering_fields`. No `FilterSet` class is necessary. @@ -416,6 +415,41 @@ paths: aggregates: { $ref: '#/components/schemas/EventAggregates' } results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } + /events/{uuid}/: + get: + tags: [status] + summary: One event + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + responses: + '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/ServiceEventDetail' } } } } + '404': { $ref: '#/components/responses/Error' } + /events/{uuid}/updates/: + get: + tags: [status] + summary: One event's log, oldest first + description: Its own operation because a provider's log has no ceiling. + security: [] + parameters: + - { $ref: '#/components/parameters/Uuid' } + - { $ref: '#/components/parameters/Fields' } + - { $ref: '#/components/parameters/Cursor' } + - { $ref: '#/components/parameters/PageSize' } + responses: + '200': + description: OK + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/Envelope' + - type: object + properties: + aggregates: { $ref: '#/components/schemas/Aggregates' } + results: { type: array, items: { $ref: '#/components/schemas/EventUpdate' } } + /dashboards/{uuid}/components/: get: tags: [board] @@ -972,9 +1006,25 @@ components: Render the label from `/meta/` `enums.event_phase`, keyed by `kind`. starts_at: { type: string, format: date-time, description: When it began, or when the window opens. } ends_at: { type: string, format: date-time, nullable: true, description: When it resolved, or when the window closes. } - updates: - type: array - items: { $ref: '#/components/schemas/EventUpdate' } + detected_by: + type: string + enum: [provider, system] + description: | + Who opened it: the provider's own page, or this deployment noticing a severity drop + first. Never rewritten by a later claim. Render the label from `/meta/` + `enums.event_source`. + service: { $ref: '#/components/schemas/ServiceRef' } + ServiceEventDetail: + allOf: + - $ref: '#/components/schemas/ServiceEvent' + - type: object + description: | + The event screen's header. Both counts are here because the header draws both tab + badges before either tab has made a request. + properties: + update_count: { type: integer } + affected_count: { type: integer } + last_update_at: { type: string, format: date-time, nullable: true } EventUpdate: type: object description: One posted update within an event's timeline. diff --git a/docs/check_docs.py b/docs/check_docs.py index e551369..37cc1dd 100755 --- a/docs/check_docs.py +++ b/docs/check_docs.py @@ -34,6 +34,7 @@ "PathNode": "ServiceComponent", "Status": "ComponentStatus", "ServiceEvent": "ServiceEvent", + "ServiceEventDetail": "ServiceEvent", "EventRef": "ServiceEvent", "EventUpdate": "EventUpdate", "Me": "User", @@ -67,10 +68,12 @@ "Component.upcoming_maintenance_count": "Count of unfinished maintenance across that M2M", "Component.active_incident": "newest unresolved ServiceEvent kind=incident across that M2M", "Component.active_incident_count": "Count of unresolved across that M2M", - "ServiceEvent.updates": "reverse FK from EventUpdate", "Me.default_dashboard_id": "Dashboard where is_default", "Service.in_catalog_since": "BaseModel.created_at", "Component.service": "ForeignKey", + "ServiceEventDetail.update_count": "Count of EventUpdate for this event", + "ServiceEventDetail.affected_count": "Count of affected_components for this event", + "ServiceEventDetail.last_update_at": "newest EventUpdate.posted_at for this event", } INHERITED = {"id", "created_at", "updated_at", "created_by", "updated_by"} From 4c29d8bbc00770156a764e43aeaf68cb7249f455 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 18:11:58 -0700 Subject: [PATCH 16/94] feat: a dead end can hand us the address The Add-by-URL not-found screen offers to send us a URL, and an import stored nothing about an attempt that failed. One row per URL with a count, so the admin list is ordered by what people actually want. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/defaults.py | 5 + .../migrations/0006_service_request.py | 66 ++++++++++ api/catalog/models.py | 27 ++++ api/catalog/serializers.py | 6 + api/catalog/urls.py | 7 +- api/catalog/views.py | 50 +++++++- api/tests/conftest.py | 5 + api/tests/test_service_requests.py | 118 ++++++++++++++++++ docs/api/openapi.yaml | 28 +++++ docs/specs/2026-08-23-statusboard-design.md | 6 + 10 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 api/catalog/migrations/0006_service_request.py create mode 100644 api/tests/test_service_requests.py diff --git a/api/api/defaults.py b/api/api/defaults.py index 37315e1..7b452dd 100644 --- a/api/api/defaults.py +++ b/api/api/defaults.py @@ -86,6 +86,10 @@ class Throttle(StrEnum): SIGNED_IN = "user" IMPORT = "import" MAGIC_LINK = "magic-link" + # Its own bucket, not IMPORT's. Add-by-URL tries an import, and on + # a miss sends the same URL here. One scope would let the first + # call spend the second's budget. + SERVICE_REQUEST = "service-request" # Reading the catalog is the point, so the plain rates are wide. Each of @@ -96,6 +100,7 @@ class Throttle(StrEnum): Throttle.SIGNED_IN: "600/min", Throttle.IMPORT: "6/min", Throttle.MAGIC_LINK: "5/hour", + Throttle.SERVICE_REQUEST: "6/min", } diff --git a/api/catalog/migrations/0006_service_request.py b/api/catalog/migrations/0006_service_request.py new file mode 100644 index 0000000..3f08780 --- /dev/null +++ b/api/catalog/migrations/0006_service_request.py @@ -0,0 +1,66 @@ +# Generated by Django 5.2.17 on 2026-09-04 01:02 + +import uuid + +import django.db.models.deletion +import django.utils.timezone +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0005_component_search_document"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ServiceRequest", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid7, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("url", models.URLField(unique=True)), + ("request_count", models.PositiveIntegerField(default=1)), + ( + "last_requested_at", + models.DateTimeField(default=django.utils.timezone.now), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["-request_count", "-last_requested_at"], + "abstract": False, + }, + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index bb004c1..589c689 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -204,6 +204,33 @@ def normalise_url(url: str) -> str: ) +class ServiceRequest(BaseModel): + """A status page somebody asked for and we could not read. + + One row per URL, not one per request. A row per request would + answer one question twice once a URL was triaged. + + There is no `requested_by`. `BaseModel.created_by` already holds + it, null when the asker was signed out. The v1 spec deleted + `Service.added_by` for the same reason. + + There is no state column either. Nothing closes a request yet, so + it would have no writer. + """ + + url = models.URLField(unique=True) + # The demand signal. Nothing else records how often a URL was + # asked for. This is that record, not a copy of one. + request_count = models.PositiveIntegerField(default=1) + last_requested_at = models.DateTimeField(default=timezone.now) + + class Meta(BaseModel.Meta): + ordering = ["-request_count", "-last_requested_at"] + + def __str__(self): + return self.url + + def _descendant_count(): """How many components sit under each row, in one query for the page. diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index cf0515c..14bf0fb 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -222,3 +222,9 @@ def validate_status_page_url(self, value): except BlockedAddress as error: raise serializers.ValidationError(str(error)) from error return value + + +class ServiceRequestSerializer(serializers.Serializer): + """The body of POST /catalog/requests/.""" + + url = serializers.URLField() diff --git a/api/catalog/urls.py b/api/catalog/urls.py index 69972c8..a440eae 100644 --- a/api/catalog/urls.py +++ b/api/catalog/urls.py @@ -1,7 +1,7 @@ from django.urls import include, path from rest_framework.routers import SimpleRouter -from catalog.views import CatalogImportView, ServiceViewSet +from catalog.views import CatalogImportView, ServiceRequestView, ServiceViewSet from catalog.views_components import ComponentDetailView, ComponentListView router = SimpleRouter(trailing_slash=True) @@ -12,6 +12,11 @@ urlpatterns = [ # Above the router so its slug route does not shadow it. path("catalog/import/", CatalogImportView.as_view(), name="catalog-import"), + path( + "catalog/requests/", + ServiceRequestView.as_view(), + name="catalog-request", + ), path( "catalog/components/", ComponentListView.as_view(), diff --git a/api/catalog/views.py b/api/catalog/views.py index 07afa5f..f89cfcc 100644 --- a/api/catalog/views.py +++ b/api/catalog/views.py @@ -1,4 +1,6 @@ import requests +from django.db.models import F +from django.utils import timezone from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.utils import OpenApiResponse, extend_schema from rest_framework import status as http @@ -11,11 +13,12 @@ from api.defaults import Throttle from catalog.filters import ComponentFilter, ServiceEventFilter, ServiceFilter -from catalog.models import Service, ServiceComponent +from catalog.models import Service, ServiceComponent, ServiceRequest, StatusPage from catalog.queries import OVERALL_SEVERITY, WATCHER_COUNT from catalog.serializers import ( ComponentSerializer, ImportRequestSerializer, + ServiceRequestSerializer, ServiceSerializer, ) from common.aggregates import EventAggregateSet, StatusAggregateSet @@ -173,3 +176,48 @@ def post(self, request): ServiceSerializer(service, context={"request": request}).data, status=http.HTTP_201_CREATED if created else http.HTTP_200_OK, ) + + +class ServiceRequestView(APIView): + """ "Send this URL to us", from the Add-by-URL not-found screen. + + An import stores nothing about an attempt that failed, so this is + where a dead end goes. Telling somebody to hunt for a better link + assumes they have not already tried. + """ + + permission_classes = [AllowAny] + # An anonymous write. Without this one person could inflate the + # demand signal the admin list is ordered by. + # + # Its own scope, not IMPORT's. A failed import and the report that + # follows it share one flow. One bucket would let the import spend + # the report's budget. + throttle_scope = Throttle.SERVICE_REQUEST + + @extend_schema( + request=ServiceRequestSerializer, + responses={ + 202: OpenApiResponse(description="Recorded."), + 400: OpenApiResponse(description="Missing or malformed url."), + }, + ) + def post(self, request): + body = ServiceRequestSerializer(data=request.data) + if not body.is_valid(): + return Response(body.errors, status=400) + url = StatusPage.normalise_url(body.validated_data["url"]) + author = request.user if request.user.is_authenticated else None + row, created = ServiceRequest.objects.get_or_create( + url=url, defaults={"created_by": author, "updated_by": author} + ) + if not created: + # F() rather than a read and a write. Two people asking at + # once would otherwise both write 2. + ServiceRequest.objects.filter(pk=row.pk).update( + request_count=F("request_count") + 1, + last_requested_at=timezone.now(), + ) + # Always 202, and always the same body. Anything else would + # tell a stranger which URLs the catalog already holds. + return Response(status=http.HTTP_202_ACCEPTED) diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 5cf0f90..7bd36c2 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -30,6 +30,11 @@ def authenticated_client(board): return jwt_client(board.owner) +@pytest.fixture +def user(db): + return User.objects.create(email="asker@example.test") + + @pytest.fixture(autouse=True) def no_network(monkeypatch): """Fail loudly if any test reaches the internet. diff --git a/api/tests/test_service_requests.py b/api/tests/test_service_requests.py new file mode 100644 index 0000000..6b26af3 --- /dev/null +++ b/api/tests/test_service_requests.py @@ -0,0 +1,118 @@ +import pytest +from django.urls import reverse + +from catalog.models import ServiceRequest +from tests.conftest import jwt_client + +pytestmark = pytest.mark.django_db + + +def test_asking_records_the_url(client): + # The write itself. Regressing here loses the row, and the admin + # list this endpoint feeds has nothing to show. + response = client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert response.status_code == 202 + row = ServiceRequest.objects.get() + assert row.url == "https://status.fastmail.com" + assert row.request_count == 1 + # Anonymous stays anonymous. A future edit that stopped guarding + # `is_authenticated` would only fail here, not on save. + assert row.created_by is None + + +def test_asking_twice_counts_rather_than_duplicates(client): + # State belongs to the URL, not to a request. A row per request + # would answer one question twice once a URL was triaged. + for _ in range(2): + client.post( + reverse("catalog-request"), + {"url": "https://status.fastmail.com/"}, + content_type="application/json", + ) + assert ServiceRequest.objects.count() == 1 + assert ServiceRequest.objects.get().request_count == 2 + + +def test_two_spellings_of_one_url_are_one_row(client): + # `url` is unique on the normalised form. Skipping normalisation + # before the lookup would let a trailing slash duplicate the row + # `normalise_url` exists to prevent. + client.post( + reverse("catalog-request"), + {"url": "https://status.example.com/"}, + content_type="application/json", + ) + client.post( + reverse("catalog-request"), + {"url": "https://status.example.com"}, + content_type="application/json", + ) + assert ServiceRequest.objects.count() == 1 + assert ServiceRequest.objects.get().request_count == 2 + + +def test_the_answer_is_the_same_whether_we_hold_it_or_not(client): + # Always 202. A different code would reveal which URLs are + # already in the catalog to anyone who asked. + first = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + second = client.post( + reverse("catalog-request"), + {"url": "https://status.one.example/"}, + content_type="application/json", + ) + assert first.status_code == second.status_code == 202 + assert first.content == second.content + + +def test_a_signed_in_asker_is_recorded(user): + # created_by carries it. There is no requested_by: that is what + # BaseModel already holds. `force_login` leaves this view + # answering 401, because there is no session backend. + jwt_client(user).post( + reverse("catalog-request"), + {"url": "https://status.two.example/"}, + format="json", + ) + assert ServiceRequest.objects.get().created_by == user + + +def test_a_malformed_url_is_refused(client): + # Refused before it is stored. Skipping validation would let junk + # into the row the admin list is ordered by. + response = client.post( + reverse("catalog-request"), + {"url": "not a url"}, + content_type="application/json", + ) + assert response.status_code == 400 + + +def test_a_failed_import_does_not_spend_this_endpoints_budget(client, monkeypatch): + # Add-by-URL tries the import, then reports the same URL here. One + # shared scope would let the import spend this endpoint's budget. + # An asker who had already tried six times could never send us + # the URL that failed. + def refuses(url): + raise ValueError(f"no adapter for {url}") + + monkeypatch.setattr("catalog.views.import_from_url", refuses) + for _ in range(6): + client.post( + reverse("catalog-import"), + {"status_page_url": "https://status.example.com/"}, + content_type="application/json", + ) + response = client.post( + reverse("catalog-request"), + {"url": "https://status.example.com/"}, + content_type="application/json", + ) + assert response.status_code == 202 diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index a1c3347..6f3d847 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -247,6 +247,34 @@ paths: description: No supported status page found at that URL content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } + /catalog/requests/: + post: + tags: [catalog] + summary: Ask us for a status page we could not read + description: | + The Add-by-URL not-found screen. Always `202`, whether or not the catalog already + holds the URL, so asking cannot reveal what we have. + + One row per URL, counted. Throttled per caller. + security: [] + requestBody: + required: true + content: + application/json: + schema: { type: object, required: [url], properties: { url: { type: string, format: uri } } } + responses: + '202': { description: Recorded } + '400': + description: | + Missing or malformed url. A field-errors object, one array of messages per + field name — not the `Error` shape, which this response never raises. + content: + application/json: + schema: + type: object + additionalProperties: { type: array, items: { type: string } } + '429': { $ref: '#/components/responses/Throttled' } + /catalog/components/: get: tags: [catalog] diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 9086ec8..f9f7a3c 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -201,6 +201,12 @@ erDiagram bool ok text error } + ServiceRequest { + uuid id PK + url url UK "normalised" + int request_count "the demand signal" + datetime last_requested_at + } User ||--|| Dashboard : owns Dashboard ||--o{ DashboardItem : holds From 8c2870c4672add152f3db570ae9e6c94e0816e85 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 18:45:56 -0700 Subject: [PATCH 17/94] refactor: delete what nothing reads any more The service list, both nested routes, ServiceFilter, WATCHER_COUNT, Service.description and Service.is_featured all served screens the approved decks replaced. by_event_kind counted tabs that no longer exist, and the board's event filter named the same two. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 25 +- api/catalog/filters.py | 49 +--- .../migrations/0007_drop_service_columns.py | 28 +++ api/catalog/models.py | 9 +- api/catalog/queries.py | 18 +- api/catalog/serializers.py | 1 - api/catalog/urls.py | 18 +- api/catalog/views.py | 128 ++-------- api/common/aggregates.py | 46 +--- api/common/filters.py | 26 ++ api/dashboards/filters.py | 33 +-- api/dashboards/views.py | 14 +- api/polling/adapters/services/cstate.py | 5 +- api/polling/importer.py | 1 - api/polling/tasks.py | 2 +- api/tests/test_admin.py | 1 - api/tests/test_board_api.py | 101 +++----- api/tests/test_catalog_api.py | 225 ++++++++---------- api/tests/test_common_api.py | 2 +- api/tests/test_contract.py | 31 --- api/tests/test_event_api.py | 49 ++++ api/tests/test_import.py | 9 +- api/tests/test_security.py | 2 +- docs/api/openapi.yaml | 139 +---------- docs/specs/2026-08-23-statusboard-design.md | 45 ++-- 25 files changed, 320 insertions(+), 687 deletions(-) create mode 100644 api/catalog/migrations/0007_drop_service_columns.py diff --git a/api/catalog/admin.py b/api/catalog/admin.py index c5c3351..2f1d46a 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -19,7 +19,7 @@ from unfold.forms import BaseDialogForm from catalog.models import Service, ServiceComponent, StatusPage -from catalog.queries import OVERALL_SEVERITY, WATCHER_COUNT +from catalog.queries import OVERALL_SEVERITY, is_tracked from common.admin import ( SEVERITY_VARIANTS, BaseModelAdmin, @@ -242,8 +242,8 @@ def has_add_permission(self, request, obj=None): class TrackedFilter(DropdownFilter): """Whether anybody has this service on a board. - The count is worked out when the list is read, so there is no column - to put a range over. This is the question the poller asks too. + There is no column to filter on. `is_tracked` is the question the + poller asks, so the admin asks it the same way. """ title = _("Tracked") @@ -255,9 +255,8 @@ def lookups(self, request, model_admin): def queryset(self, request, queryset): if self.value() is None: return queryset - if self.value() == "1": - return queryset.filter(watcher_count__gt=0) - return queryset.filter(watcher_count=0) + watched = is_tracked() + return queryset.filter(watched if self.value() == "1" else ~watched) class ServiceSeverityFilter(DropdownFilter): @@ -310,15 +309,12 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): list_display = [ "display_service", "display_severity", - "is_featured", - "display_watchers", "provider", "display_related", ] search_fields = [ "name", "slug", - "description", "homepage_url", "status_page__url", ] @@ -327,17 +323,14 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): # when. A service belongs to nothing, so its state leads. list_filter = [ ServiceSeverityFilter, - "is_featured", ("status_page__provider", ChoicesDropdownFilter), TrackedFilter, ("created_at", RangeDateTimeFilter), ("updated_at", RangeDateTimeFilter), ] - # Not `ordering`: the checks want a real column there, and this one - # is counted when the list is read. `get_queryset` sorts instead. ordering = ["name"] fieldsets = [ - (None, {"fields": ["name", "slug", "description", "is_featured"]}), + (None, {"fields": ["name", "slug"]}), (_("Presentation"), {"fields": ["logo", "homepage_url"]}), audit_section(), ] @@ -361,20 +354,14 @@ def get_queryset(self, request): .select_related("status_page", "poller") .annotate( severity_now=OVERALL_SEVERITY, - watcher_count=WATCHER_COUNT, component_count=related_count(ServiceComponent.objects, "service"), event_count=related_count(ServiceEvent.objects, "service"), status_count=related_count( ComponentStatus.objects, "component__service" ), ) - .order_by("-watcher_count", "name") ) - @display(description=_("Watchers"), ordering="watcher_count") - def display_watchers(self, obj): - return obj.watcher_count - @display(description=_("Service"), header=True, ordering="name") def display_service(self, obj): # Two lines and the product's own mark, so a long catalog diff --git a/api/catalog/filters.py b/api/catalog/filters.py index 9fcd2d9..c5eee74 100644 --- a/api/catalog/filters.py +++ b/api/catalog/filters.py @@ -7,42 +7,11 @@ from django_filters import rest_framework as filters -from catalog.models import Service, ServiceComponent -from status.models import ServiceEvent +from catalog.models import ServiceComponent +from common.filters import SeverityFilterMixin -class ServiceFilter(filters.FilterSet): - # Declared for a second reason. `overall_component` is not a relation - # on Service. The current severity is not a column either. It is the - # open row of a component's status history. So the contract's name is - # kept, pointed at the view's `severity_now` annotation. - overall_component__status__severity = filters.NumberFilter( - field_name="severity_now" - ) - overall_component__status__severity__lte = filters.NumberFilter( - field_name="severity_now", lookup_expr="lte" - ) - - class Meta: - model = Service - fields = { - "status_page__provider": ["exact"], - "is_featured": ["exact"], - } - - -class ComponentFilter(filters.FilterSet): - # `status` is not a relation. A component has a history of statuses - # and the open one is current. Same contract name, same annotation. - status__severity = filters.NumberFilter(field_name="severity_now") - status__severity__lte = filters.NumberFilter( - field_name="severity_now", lookup_expr="lte" - ) - # The Severity filter offers all six values at once. One exact match - # cannot serve it. - status__severity__in = filters.BaseInFilter( - field_name="severity_now", lookup_expr="in" - ) +class ComponentFilter(SeverityFilterMixin): service = filters.CharFilter(field_name="service__slug") # Every descendant, not one level. `parent` would name a query this # does not run. @@ -75,15 +44,3 @@ def filter_is_tracked(self, queryset, name, value): if self.request is None or self.request.user.is_anonymous: return queryset.none() if value else queryset return queryset.filter(_is_tracked=value) - - -class ServiceEventFilter(filters.FilterSet): - class Meta: - model = ServiceEvent - fields = { - "kind": ["exact"], - "phase": ["exact", "in"], - "ends_at": ["isnull", "gte"], - "starts_at": ["gte", "lte"], - "affected_components": ["exact"], - } diff --git a/api/catalog/migrations/0007_drop_service_columns.py b/api/catalog/migrations/0007_drop_service_columns.py new file mode 100644 index 0000000..3a699fd --- /dev/null +++ b/api/catalog/migrations/0007_drop_service_columns.py @@ -0,0 +1,28 @@ +# Generated by Django 5.2.17 on 2026-09-04 01:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0006_service_request"), + ] + + operations = [ + migrations.RemoveField( + model_name="historicalservice", + name="description", + ), + migrations.RemoveField( + model_name="historicalservice", + name="is_featured", + ), + migrations.RemoveField( + model_name="service", + name="description", + ), + migrations.RemoveField( + model_name="service", + name="is_featured", + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index 589c689..f279c0d 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -17,10 +17,11 @@ class ServiceQuerySet(models.QuerySet): def for_display(self, user=None): - """Everything `ServiceSerializer` reads, without a query a row. + """Everything `ServiceSerializer` reads, in four queries. - Three fields asked per service, and one of them serialized a - component, which asked seven more. + The service page reads one service. Three of its fields are + counted or nested. The nested one is a whole component, which + asks seven more. """ from django.contrib.auth.models import AnonymousUser @@ -59,10 +60,8 @@ class Service(BaseModel): blank=True, help_text="Derived from the name when left blank.", ) - description = models.TextField(blank=True, default="") logo = models.URLField(blank=True, default="") homepage_url = models.URLField(blank=True, default="") - is_featured = models.BooleanField(verbose_name="Featured", default=False) objects = ServiceQuerySet.as_manager() history = HistoricalRecords() diff --git a/api/catalog/queries.py b/api/catalog/queries.py index 26df89b..609b2b9 100644 --- a/api/catalog/queries.py +++ b/api/catalog/queries.py @@ -1,7 +1,7 @@ """What a caller reads about a service, as SQL. -A service's own severity is its overall component's. Who watches it is -counted through the boards its components sit on. Both are questions +A service's own severity is its overall component's. Who watches a +component is counted through the boards it sits on. Both are questions about the catalog, so they live here. These lived in `common.ordering`, which made the base layer import the @@ -22,16 +22,10 @@ ).values("severity")[:1] ) -# Who tracks a service, counted when it is read. It was a column that a -# signal kept true, and four write paths never reached the signal. -# -# Two readers, two shapes. Only the suggestion order needs the number, so -# only it pays for a distinct count over three joins. -WATCHER_COUNT = Count("components__boards__owner", distinct=True) - -# Who tracks one component. `DashboardItem` points straight at it, so -# this is one join. The service version needed three, and only the -# service sort read it. +# Who tracks one component, counted when it is read. It was a column +# that a signal kept true, and four write paths never reached the +# signal. `DashboardItem` points straight at a component, so this is one +# join. COMPONENT_WATCHER_COUNT = Count("boards__owner", distinct=True) diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index 14bf0fb..dc5c18e 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -166,7 +166,6 @@ class Meta: "id", "slug", "name", - "description", "homepage_url", "logo", "in_catalog_since", diff --git a/api/catalog/urls.py b/api/catalog/urls.py index a440eae..44df4a7 100644 --- a/api/catalog/urls.py +++ b/api/catalog/urls.py @@ -1,22 +1,16 @@ -from django.urls import include, path -from rest_framework.routers import SimpleRouter +from django.urls import path -from catalog.views import CatalogImportView, ServiceRequestView, ServiceViewSet +from catalog.views import CatalogImportView, ServiceDetailView, ServiceRequestView from catalog.views_components import ComponentDetailView, ComponentListView -router = SimpleRouter(trailing_slash=True) -# basename "service" gives service-list, service-detail, and one route per -# detail action: service-components and service-events. -router.register("catalog/services", ServiceViewSet, basename="service") - urlpatterns = [ - # Above the router so its slug route does not shadow it. path("catalog/import/", CatalogImportView.as_view(), name="catalog-import"), path( "catalog/requests/", ServiceRequestView.as_view(), name="catalog-request", ), + # Above the service route, so `components` is never read as a slug. path( "catalog/components/", ComponentListView.as_view(), @@ -27,5 +21,9 @@ ComponentDetailView.as_view(), name="component-detail", ), - path("", include(router.urls)), + path( + "catalog/services//", + ServiceDetailView.as_view(), + name="service-detail", + ), ] diff --git a/api/catalog/views.py b/api/catalog/views.py index f89cfcc..ea84557 100644 --- a/api/catalog/views.py +++ b/api/catalog/views.py @@ -1,149 +1,51 @@ import requests from django.db.models import F from django.utils import timezone -from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.utils import OpenApiResponse, extend_schema +from rest_framework import generics from rest_framework import status as http -from rest_framework.decorators import action -from rest_framework.filters import SearchFilter from rest_framework.permissions import AllowAny from rest_framework.response import Response from rest_framework.views import APIView -from rest_framework.viewsets import ReadOnlyModelViewSet from api.defaults import Throttle -from catalog.filters import ComponentFilter, ServiceEventFilter, ServiceFilter -from catalog.models import Service, ServiceComponent, ServiceRequest, StatusPage -from catalog.queries import OVERALL_SEVERITY, WATCHER_COUNT +from catalog.models import Service, ServiceRequest, StatusPage from catalog.serializers import ( - ComponentSerializer, ImportRequestSerializer, ServiceRequestSerializer, ServiceSerializer, ) -from common.aggregates import EventAggregateSet, StatusAggregateSet from common.errors import NoStatusPageFound, ProviderUnreachable from common.filters import FieldsBackend -from common.ordering import MappedOrderingFilter from polling.importer import import_from_url -from status.models import ServiceEvent -from status.queries import CURRENT_SEVERITY -from status.serializers import ServiceEventSerializer -class ServiceAggregateSet(StatusAggregateSet): - """Counts for a list of services. +class ServiceDetailView(generics.RetrieveAPIView): + """The service page's header and its About tab. - A service has no severity of its own. Its severity is the open status - of its overall component, which is the provider's own page-level - reading. So counting services by severity means counting those - components, not the services. - """ - - def components(self): - return ServiceComponent.objects.filter( - service__in=self.queryset, is_overall=True - ) - - -class ServiceViewSet(ReadOnlyModelViewSet): - """Every read of a service, in one place. - - The list and the detail are the same resource. A component list and - an event list belong to a service, so they are detail actions. - - Each action overrides the serializer, filters and ordering it needs. - Those names exist on the class because DRF refuses an initkwarg that - is not already an attribute. + There is no list. Discover searches components, and the signed-out + board lists overall components. So nothing asked a service + collection a question. """ permission_classes = [AllowAny] lookup_field = "slug" serializer_class = ServiceSerializer - aggregate_set = ServiceAggregateSet - filterset_class = ServiceFilter - filter_backends = [ - DjangoFilterBackend, - SearchFilter, - MappedOrderingFilter, - FieldsBackend, - ] - search_fields = ["name", "slug"] - ordering_fields = ["name", "updated_at"] - # `suggested` is not a field. Severity sits behind a related path. - # - # Severity ahead of popularity is deliberate. A middling service - # that is broken now beats a popular one that is fine. - # - # Lower severity is worse, so ascending puts the broken first. A - # service with no reading sorts last, not as healthy. - SUGGESTED = ["-is_featured", "severity_now", "-watcher_count", "name"] - ordering_map = { - "suggested": SUGGESTED, - "overall_component__status__severity": ["severity_now"], - } - ordering = SUGGESTED - # Schema generation calls get_queryset with no URL kwargs. This names - # the model without running it. + filter_backends = [FieldsBackend] + # get_queryset reads the request, which schema generation has not + # set. This names the model without running it. queryset = Service.objects.none() def get_queryset(self): - # `q` is SearchFilter's, through SEARCH_PARAM. It was a hand - # rolled `name__icontains` beside a `search_fields` no backend - # read. The slug was declared searchable, and was not. - return Service.objects.for_display(self.request.user).annotate( - watcher_count=WATCHER_COUNT, - severity_now=OVERALL_SEVERITY, - ) - - def _page(self, queryset): - """The same envelope the list actions return.""" - page = self.paginate_queryset(self.filter_queryset(queryset)) - return self.get_paginated_response(self.get_serializer(page, many=True).data) - - @extend_schema(responses={200: ComponentSerializer}) - @action( - detail=True, - url_path="components", - url_name="components", - serializer_class=ComponentSerializer, - aggregate_set=StatusAggregateSet, - filterset_class=ComponentFilter, - ordering_fields=["name", "status_page_order", "updated_at"], - ordering_map={"status__severity": ["severity_now"]}, - ordering=["status_page_order"], - ) - def components(self, request, slug=None): - return self._page( - ServiceComponent.objects.filter(service__slug=slug) - .for_display(self.request.user) - .annotate(severity_now=CURRENT_SEVERITY) - ) - - @extend_schema(responses={200: ServiceEventSerializer}) - @action( - detail=True, - url_path="events", - url_name="events", - serializer_class=ServiceEventSerializer, - aggregate_set=EventAggregateSet, - filterset_class=ServiceEventFilter, - ordering_fields=["starts_at", "ends_at"], - ordering_map={}, - ordering=["-starts_at"], - ) - def events(self, request, slug=None): - return self._page( - ServiceEvent.objects.filter(service__slug=slug).select_related("service") - ) + return Service.objects.for_display(self.request.user) class CatalogImportView(APIView): - """`POST /catalog/services/` is reserved. + """Add a service by pasting the address of its status page. - A future admin create takes a service body. A bulk create takes a - list. So importing by URL gets its own path. It does not overload - the standard collection. + Its own path, because the body is a URL rather than a service. A + later create that takes a service body then does not overload this + one. """ permission_classes = [AllowAny] diff --git a/api/common/aggregates.py b/api/common/aggregates.py index 5d868f5..65f05d2 100644 --- a/api/common/aggregates.py +++ b/api/common/aggregates.py @@ -1,6 +1,4 @@ -from django.db.models import Count, Exists, Min, OuterRef - -from status.choices import EventKind +from django.db.models import Count, Min class AggregateSet: @@ -22,62 +20,28 @@ def build(self): class StatusAggregateSet(AggregateSet): - """For any list of things carrying a status. Services, or a board.""" + """For any list of components. Discover, a service's parts, a board.""" def build(self): data = super().build() data["by_severity"] = self._by_severity() - data["by_event_kind"] = self._by_event_kind() data["next_refresh_at"] = self._next_refresh_at() data["oldest_refreshed_at"] = self._oldest_refreshed_at() return data - def components(self): - """The components these counts are drawn from. - - Severity and events hang off a component, so every count below - starts here. When the rows are already components this is them. - A list of something else overrides it to say which components - stand for its rows. - """ - return self.queryset - def _by_severity(self): rows = ( - self.components() - .filter(statuses__ended_at__isnull=True) + self.queryset.filter(statuses__ended_at__isnull=True) .values("statuses__severity") .annotate(n=Count("id", distinct=True)) ) return {str(r["statuses__severity"]): r["n"] for r in rows} - def _by_event_kind(self): - """How many components have a live event of each kind. - - Through `ServiceEvent.objects.live`, the same rule the component - beside it counts by. This had its own copy, and it left out the - end of a maintenance window. A chip and a row could disagree - about one that was over. - """ - from status.models import ServiceEvent - - counts = {} - for kind in EventKind: - live = ServiceEvent.objects.live(kind).filter( - affected_components=OuterRef("pk") - ) - counts[str(kind)] = ( - self.components().filter(Exists(live)).distinct().count() - ) - return counts - def _next_refresh_at(self): - return self.components().aggregate(v=Min("service__poller__next_at"))["v"] + return self.queryset.aggregate(v=Min("service__poller__next_at"))["v"] def _oldest_refreshed_at(self): - return self.components().aggregate(v=Min("service__poller__last_success_at"))[ - "v" - ] + return self.queryset.aggregate(v=Min("service__poller__last_success_at"))["v"] class EventAggregateSet(AggregateSet): diff --git a/api/common/filters.py b/api/common/filters.py index 29ae67d..8cad26a 100644 --- a/api/common/filters.py +++ b/api/common/filters.py @@ -1,3 +1,4 @@ +from django_filters import rest_framework as filters from rest_framework.filters import BaseFilterBackend FIELDS_DESCRIPTION = ( @@ -27,3 +28,28 @@ def get_schema_operation_parameters(self, view): "schema": {"type": "string"}, } ] + + +class SeverityFilterMixin(filters.FilterSet): + """How a caller narrows a list of components by severity. + + Declared, not generated. A component has a history of statuses and + the open one is current. So there is no `status` relation to + generate from. The contract's name points at the `severity_now` + annotation each view supplies instead. + + A `FilterSet` rather than a plain mixin. django-filter collects + declared filters only from a base that already carries them. It + names no model, so `common` still imports no app. A subclass brings + its own `Meta`. + """ + + status__severity = filters.NumberFilter(field_name="severity_now") + status__severity__lte = filters.NumberFilter( + field_name="severity_now", lookup_expr="lte" + ) + # The Severity filter offers all six values at once. One exact match + # cannot serve it. + status__severity__in = filters.BaseInFilter( + field_name="severity_now", lookup_expr="in" + ) diff --git a/api/dashboards/filters.py b/api/dashboards/filters.py index d5271a9..0a6dc9f 100644 --- a/api/dashboards/filters.py +++ b/api/dashboards/filters.py @@ -1,37 +1,8 @@ -from django.db.models import Exists, OuterRef -from django_filters import rest_framework as filters - from catalog.models import ServiceComponent -from status.choices import CLOSED_PHASES, EventKind -from status.models import ServiceEvent - - -class BoardComponentFilter(filters.FilterSet): - """`event` is declared, not generated. +from common.filters import SeverityFilterMixin - `ServiceEvent` and `ServiceComponent` are many-to-many. - django-filter applies each parameter as its own `.filter()` call. - On a many-to-many, each call joins again and can match a different event. - Both conditions must sit in one call, so this is a single subquery. - """ - - event = filters.ChoiceFilter(choices=EventKind.choices, method="filter_event") - - # Declared for the same reason as the catalog's. A component has a - # history of `statuses`. The open one is current. So there is no - # `status` relation to generate from. The contract's name points at - # the annotation instead. - status__severity = filters.NumberFilter(field_name="severity_now") - status__severity__lte = filters.NumberFilter( - field_name="severity_now", lookup_expr="lte" - ) +class BoardComponentFilter(SeverityFilterMixin): class Meta: model = ServiceComponent fields = [] - - def filter_event(self, queryset, name, value): - live = ServiceEvent.objects.filter( - kind=value, affected_components=OuterRef("pk") - ).exclude(phase__in=CLOSED_PHASES) - return queryset.filter(Exists(live)) diff --git a/api/dashboards/views.py b/api/dashboards/views.py index 4c5ec93..c24f3d3 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -23,8 +23,8 @@ def _board(request, uuid): class BoardComponentListView(generics.ListCreateAPIView): - # get_queryset reads self.kwargs, which schema generation has - # not set. This names the model without running it. + # Schema generation reads the model off this before it calls + # get_queryset. It names the model without running one. queryset = ServiceComponent.objects.none() permission_classes = [IsAuthenticated] serializer_class = ComponentSerializer @@ -40,11 +40,17 @@ class BoardComponentListView(generics.ListCreateAPIView): ordering = ["severity_now"] # worst first: lower severity is worse def get_queryset(self): + rows = ServiceComponent.objects.annotate( + severity_now=CURRENT_SEVERITY, next_transition=NEXT_TRANSITION + ) + if "uuid" not in self.kwargs: + # Schema generation reaches here with no URL. It reads the + # annotations, because the filters and the sorts name them. + return rows.none() board = _board(self.request, self.kwargs["uuid"]) return ( - ServiceComponent.objects.filter(tracked_by__dashboard=board) + rows.filter(tracked_by__dashboard=board) .for_display(self.request.user) - .annotate(severity_now=CURRENT_SEVERITY, next_transition=NEXT_TRANSITION) .distinct() ) diff --git a/api/polling/adapters/services/cstate.py b/api/polling/adapters/services/cstate.py index 6f1b1d1..71d7c8a 100644 --- a/api/polling/adapters/services/cstate.py +++ b/api/polling/adapters/services/cstate.py @@ -75,7 +75,4 @@ def fetch_incidents(self): def fetch_service_metadata(self): payload = self._payload() - return { - "name": payload.get("title", ""), - "description": payload.get("description", ""), - } + return {"name": payload.get("title", "")} diff --git a/api/polling/importer.py b/api/polling/importer.py index 983c0aa..c8c4b19 100644 --- a/api/polling/importer.py +++ b/api/polling/importer.py @@ -76,7 +76,6 @@ def _import(key, adapter_class, fetch_url, fetched): author = get_user_model().objects.system() service = Service.objects.create( name=metadata["name"], - description=metadata.get("description", ""), homepage_url=metadata.get("homepage_url", ""), logo=fetched["logo"], created_by=author, diff --git a/api/polling/tasks.py b/api/polling/tasks.py index 19b1376..def78ed 100644 --- a/api/polling/tasks.py +++ b/api/polling/tasks.py @@ -75,7 +75,7 @@ def _refresh_metadata(service, metadata, adapter=None): if logo: service.logo = logo changed.append("logo") - for field in ("name", "description", "homepage_url"): + for field in ("name", "homepage_url"): value = metadata.get(field) if value and getattr(service, field) != value: setattr(service, field, value) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 2cc5dec..c3f9193 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -27,7 +27,6 @@ def service_form_data(staff_client, **fields): data = { "name": "", "slug": "", - "description": "", "logo": "", "homepage_url": "", **fields, diff --git a/api/tests/test_board_api.py b/api/tests/test_board_api.py index 4659bf2..3264d7c 100644 --- a/api/tests/test_board_api.py +++ b/api/tests/test_board_api.py @@ -8,7 +8,6 @@ from status.choices import ( EventKind, IncidentPhase, - MaintenancePhase, Severity, StatusSource, ) @@ -71,86 +70,52 @@ def test_all_returns_every_tracked_component(client, board): assert client.get(url).json()["aggregates"]["total"] == 2 -@pytest.mark.django_db -def test_the_incidents_tab_filters_on_event_kind(client, board): - with_incident = _track(board) - _event(with_incident, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) - _track(board) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=incident" - assert len(client.get(url).json()["results"]) == 1 - - -@pytest.mark.django_db -def test_a_resolved_incident_does_not_put_a_row_on_the_incidents_tab(client, board): - component = _track(board) - _event( - component, EventKind.INCIDENT, IncidentPhase.RESOLVED, ends_at=timezone.now() - ) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=incident" - assert client.get(url).json()["results"] == [] - - -@pytest.mark.django_db -def test_the_event_filter_binds_both_conditions_to_the_same_event(client, board): - # `ServiceEvent` and `ServiceComponent` are many-to-many. - # Two parameters filter twice and can match two different events. - component = _track(board) - _event( - component, EventKind.INCIDENT, IncidentPhase.RESOLVED, ends_at=timezone.now() - ) - _event(component, EventKind.MAINTENANCE, MaintenancePhase.IN_PROGRESS) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=incident" - assert client.get(url).json()["results"] == [] - - -@pytest.mark.django_db -def test_scheduled_maintenance_appears_on_the_maintenance_tab_while_reading_operational( - client, board -): - # A window three days out leaves severity 5. - # Severity cannot select it, so the tab filters on the event. - component = _track(board, severity=Severity.OPERATIONAL) - _event( - component, - EventKind.MAINTENANCE, - MaintenancePhase.SCHEDULED, - starts_at=timezone.now() + timezone.timedelta(days=3), - ) - url = reverse("board-components", kwargs={"uuid": board.id}) + "?event=maintenance" - rows = client.get(url).json()["results"] - assert len(rows) == 1 - assert rows[0]["status"]["severity"] == Severity.OPERATIONAL - - @pytest.mark.django_db def test_one_fetch_fills_every_chip(client, board): - # One response must fill all three chips. + # One response must fill every chip on the board. # Otherwise the client makes a request per chip. - broken = _track(board, severity=Severity.MAJOR_OUTAGE) - _event(broken, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) - planned = _track(board) - _event(planned, EventKind.MAINTENANCE, MaintenancePhase.SCHEDULED) + _track(board, severity=Severity.MAJOR_OUTAGE) + _track(board, severity=Severity.OPERATIONAL) aggregates = client.get( reverse("board-components", kwargs={"uuid": board.id}) ).json()["aggregates"] assert aggregates["total"] == 2 - assert aggregates["by_event_kind"]["incident"] == 1 - assert aggregates["by_event_kind"]["maintenance"] == 1 + assert aggregates["by_severity"][str(Severity.MAJOR_OUTAGE)] == 1 + assert aggregates["by_severity"][str(Severity.OPERATIONAL)] == 1 assert "next_refresh_at" in aggregates assert "oldest_refreshed_at" in aggregates @pytest.mark.django_db -def test_severity_composes_with_a_tab(client, board): - down = _track(board, severity=Severity.MAJOR_OUTAGE) - _event(down, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) - up = _track(board, severity=Severity.OPERATIONAL) - _event(up, EventKind.INCIDENT, IncidentPhase.MONITORING) - url = ( - reverse("board-components", kwargs={"uuid": board.id}) - + "?event=incident&status__severity__lte=3" +def test_the_board_takes_several_severities_at_once(client, board): + # The Severity filter offers all six values. A single exact match + # could not express "everything that needs attention". + _track(board, severity=Severity.MAJOR_OUTAGE) + _track(board, severity=Severity.PARTIAL_OUTAGE) + _track(board, severity=Severity.OPERATIONAL) + response = client.get( + reverse("board-components", kwargs={"uuid": board.id}), + {"status__severity__in": "0,1,2"}, ) - assert len(client.get(url).json()["results"]) == 1 + assert response.status_code == 200 + severities = {r["status"]["severity"] for r in response.json()["results"]} + assert severities == {Severity.MAJOR_OUTAGE, Severity.PARTIAL_OUTAGE} + + +@pytest.mark.django_db +def test_the_event_parameter_is_gone(client, board): + # It named the Home Incidents and Maintenance tabs. Home is Board + # and Updates now, and Updates is `/events/?dashboard=`. + with_incident = _track(board) + _event(with_incident, EventKind.INCIDENT, IncidentPhase.INVESTIGATING) + _track(board) + body = client.get( + reverse("board-components", kwargs={"uuid": board.id}), {"event": "incident"} + ).json() + # django-filter ignores an unknown parameter rather than failing, + # so the proof is that it no longer narrows anything. + assert body["aggregates"]["total"] == 2 + assert "by_event_kind" not in body["aggregates"] @pytest.mark.django_db diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index 4b4b56f..74f6630 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -1,5 +1,5 @@ import pytest -from django.urls import reverse +from django.urls import Resolver404, resolve, reverse from django.utils import timezone from rest_framework.test import APIClient @@ -11,7 +11,20 @@ StatusSource, ) from status.models import ComponentStatus, ServiceEvent -from tests.factories import ComponentFactory, ServiceFactory, StatusPageFactory +from tests.factories import ( + ComponentFactory, + ServiceFactory, + StatusPageFactory, + track, +) + + +def resolve_or_none(path): + """The view a path reaches, or None when nothing serves it.""" + try: + return resolve(path) + except Resolver404: + return None def _with_status(service, severity, is_overall=False, external_id=None): @@ -29,14 +42,48 @@ def _with_status(service, severity, is_overall=False, external_id=None): return component +def _service_detail(service): + return APIClient().get(reverse("service-detail", args=[service.slug])).json() + + +def test_the_service_list_is_gone(): + # Discover searches components, and the signed-out board lists + # overall components. Nothing asked a service list a question. + assert resolve_or_none("/catalog/services/") is None + + @pytest.mark.django_db -def test_the_list_is_public(): - assert APIClient().get(reverse("service-list")).status_code == 200 +def test_the_nested_component_route_is_gone(): + # `/catalog/components/?service=` serves it, and three other + # screens besides. + service = ServiceFactory() + response = APIClient().get(f"/catalog/services/{service.slug}/components/") + assert response.status_code == 404 + + +@pytest.mark.django_db +def test_the_nested_event_route_is_gone(): + # `/events/?service=` serves it, beside every other event list. + service = ServiceFactory() + response = APIClient().get(f"/catalog/services/{service.slug}/events/") + assert response.status_code == 404 + + +@pytest.mark.django_db +def test_a_service_still_answers_by_slug(): + # The service page reads it for the header and the About tab. + service = ServiceFactory(name="Twilio") + StatusPageFactory(service=service) + response = APIClient().get(reverse("service-detail", args=[service.slug])) + assert response.status_code == 200 + assert response.json()["name"] == "Twilio" + # Nothing renders a description, so it is not in the shape. + assert "description" not in response.json() @pytest.mark.django_db def test_every_list_response_uses_the_same_envelope(): - body = APIClient().get(reverse("service-list")).json() + body = APIClient().get(reverse("component-list")).json() assert set(body) == {"aggregates", "next", "results"} assert "total" in body["aggregates"] @@ -46,7 +93,7 @@ def test_a_service_nests_its_overall_component_rather_than_copying_its_fields(): service = ServiceFactory(slug="twilio") StatusPageFactory(service=service) _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True) - body = APIClient().get(reverse("service-list")).json()["results"][0] + body = _service_detail(service) assert body["overall_component"]["status"]["severity"] == Severity.MAJOR_OUTAGE assert "severity" not in body @@ -57,11 +104,7 @@ def test_the_overall_component_is_a_plain_component(): service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.OPERATIONAL, is_overall=True) - overall = ( - APIClient() - .get(reverse("service-list")) - .json()["results"][0]["overall_component"] - ) + overall = _service_detail(service)["overall_component"] for field in ( "id", "name", @@ -83,13 +126,17 @@ def test_the_overall_component_is_a_plain_component(): @pytest.mark.django_db def test_severity_filters_use_the_orm_path(): + # The contract's name is not a column. It points at an annotation, + # and a filter that missed it would answer every row. service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True) - url = reverse("service-list") + "?overall_component__status__severity__lte=3" - assert len(APIClient().get(url).json()["results"]) == 1 - url = reverse("service-list") + "?overall_component__status__severity__lte=0" - assert len(APIClient().get(url).json()["results"]) == 1 + _with_status(service, Severity.OPERATIONAL) + url = reverse("component-list") + assert len(APIClient().get(url).json()["results"]) == 2 + assert ( + len(APIClient().get(url, {"status__severity__lte": 3}).json()["results"]) == 1 + ) @pytest.mark.django_db @@ -97,11 +144,8 @@ def test_fields_prunes_the_response(): service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.OPERATIONAL, is_overall=True) - body = ( - APIClient() - .get(reverse("service-list") + "?fields=id,name") - .json()["results"][0] - ) + url = reverse("service-detail", args=[service.slug]) + body = APIClient().get(url, {"fields": "id,name"}).json() assert set(body) == {"id", "name"} @@ -110,8 +154,10 @@ def test_a_dotted_field_path_prunes_inside_a_nested_object(): service = ServiceFactory() StatusPageFactory(service=service) _with_status(service, Severity.OPERATIONAL, is_overall=True) - url = reverse("service-list") + "?fields=id,overall_component.status.severity" - body = APIClient().get(url).json()["results"][0] + url = reverse("service-detail", args=[service.slug]) + body = ( + APIClient().get(url, {"fields": "id,overall_component.status.severity"}).json() + ) assert set(body) == {"id", "overall_component"} assert set(body["overall_component"]) == {"status"} assert set(body["overall_component"]["status"]) == {"severity"} @@ -133,11 +179,7 @@ def test_active_incident_excludes_resolved_ones(): ends_at=timezone.now(), ) resolved.affected_components.add(component) - overall = ( - APIClient() - .get(reverse("service-list")) - .json()["results"][0]["overall_component"] - ) + overall = _service_detail(service)["overall_component"] assert overall["active_incident"] is None assert overall["active_incident_count"] == 0 @@ -157,128 +199,71 @@ def test_upcoming_maintenance_excludes_finished_windows(): ends_at=timezone.now() - timezone.timedelta(days=29), ) past.affected_components.add(component) - overall = ( - APIClient() - .get(reverse("service-list")) - .json()["results"][0]["overall_component"] - ) + overall = _service_detail(service)["overall_component"] assert overall["upcoming_maintenance"] is None assert overall["upcoming_maintenance_count"] == 0 -@pytest.mark.django_db -def test_the_events_endpoint_returns_both_kinds_and_filters_on_kind(): - service = ServiceFactory(slug="twilio") - StatusPageFactory(service=service) - ServiceEvent.objects.create( - service=service, - external_id="1", - kind=EventKind.INCIDENT, - title="I", - phase=IncidentPhase.INVESTIGATING, - starts_at=timezone.now(), - ) - ServiceEvent.objects.create( - service=service, - external_id="2", - kind=EventKind.MAINTENANCE, - title="M", - phase=MaintenancePhase.SCHEDULED, - starts_at=timezone.now(), - ) - url = reverse("service-events", kwargs={"slug": "twilio"}) - assert len(APIClient().get(url).json()["results"]) == 2 - assert len(APIClient().get(url + "?kind=incident").json()["results"]) == 1 - - -@pytest.mark.django_db -def test_the_events_aggregate_is_by_phase_only(): - service = ServiceFactory(slug="twilio") - StatusPageFactory(service=service) - url = reverse("service-events", kwargs={"slug": "twilio"}) - aggregates = APIClient().get(url).json()["aggregates"] - assert set(aggregates) == {"total", "by_phase"} - - -@pytest.mark.django_db -def test_an_event_carries_no_component_ids(): - # The projection runs one way: a component carries its event, not the reverse. - service = ServiceFactory(slug="twilio") - StatusPageFactory(service=service) - ServiceEvent.objects.create( - service=service, - external_id="1", - kind=EventKind.INCIDENT, - title="I", - phase=IncidentPhase.INVESTIGATING, - starts_at=timezone.now(), - ) - url = reverse("service-events", kwargs={"slug": "twilio"}) - row = APIClient().get(url).json()["results"][0] - assert set(row) == { - "id", - "kind", - "title", - "phase", - "starts_at", - "ends_at", - "detected_by", - "service", - } - - @pytest.mark.django_db def test_suggestions_put_what_is_broken_ahead_of_what_is_popular(): """Severity sits ahead of popularity on purpose. - A middling service that is broken now beats a popular one that is + A middling component that is broken now beats a popular one that is fine. That is the premise of the public view. """ - popular = ServiceFactory(name="Popular", tracked=500) + popular = ServiceFactory(name="Popular") StatusPageFactory(service=popular) - _with_status(popular, Severity.OPERATIONAL, is_overall=True) + watched = _with_status(popular, Severity.OPERATIONAL, is_overall=True) + for _ in range(5): + track(watched) - broken = ServiceFactory(name="Broken", tracked=1) + broken = ServiceFactory(name="Broken") StatusPageFactory(service=broken) - _with_status(broken, Severity.MAJOR_OUTAGE, is_overall=True) + track(_with_status(broken, Severity.MAJOR_OUTAGE, is_overall=True)) names = [ - r["name"] for r in APIClient().get(reverse("service-list")).json()["results"] + r["service"]["name"] + for r in APIClient().get(reverse("component-list")).json()["results"] ] assert names.index("Broken") < names.index("Popular") @pytest.mark.django_db -def test_a_featured_service_still_leads(): - # Featured is the cold-start seed. On day one nothing has watchers - # and nothing is polled, so it is the whole list. +def test_a_featured_component_still_leads(): + # Featured is the first key, ahead of severity. An editor's pick + # leads even when something worse is on the same page. plain = ServiceFactory(name="Plain") StatusPageFactory(service=plain) _with_status(plain, Severity.MAJOR_OUTAGE, is_overall=True) - featured = ServiceFactory(name="Featured", is_featured=True) + featured = ServiceFactory(name="Featured") StatusPageFactory(service=featured) - _with_status(featured, Severity.OPERATIONAL, is_overall=True) + row = _with_status(featured, Severity.OPERATIONAL, is_overall=True) + row.is_featured = True + row.save(update_fields=["is_featured"]) names = [ - r["name"] for r in APIClient().get(reverse("service-list")).json()["results"] + r["service"]["name"] + for r in APIClient().get(reverse("component-list")).json()["results"] ] assert names.index("Featured") < names.index("Plain") @pytest.mark.django_db -def test_a_service_with_no_reading_sorts_last_not_first(): +def test_a_component_with_no_reading_sorts_last_not_first(): # Never seen is not the same as healthy, and it must not pose as # broken either. unread = ServiceFactory(name="Unread") StatusPageFactory(service=unread) + ComponentFactory(service=unread, is_overall=True) healthy = ServiceFactory(name="Healthy") StatusPageFactory(service=healthy) _with_status(healthy, Severity.OPERATIONAL, is_overall=True) names = [ - r["name"] for r in APIClient().get(reverse("service-list")).json()["results"] + r["service"]["name"] + for r in APIClient().get(reverse("component-list")).json()["results"] ] assert names.index("Healthy") < names.index("Unread") @@ -354,31 +339,9 @@ def test_a_component_list_costs_the_same_whatever_its_length(): for size in (2, 30): reset_queries() APIClient().get( - reverse("service-components", kwargs={"slug": service.slug}), - {"page_size": size}, + reverse("component-list"), + {"service": service.slug, "page_size": size}, ) counts[size] = len(connection.queries) assert counts[2] == counts[30], counts - - -@pytest.mark.django_db -def test_a_service_list_costs_the_same_whatever_its_length(): - # Three fields asked per service, and one of them serialized a - # component, which asked seven more. - from django.db import connection, reset_queries - from django.test import override_settings - - for n in range(12): - service = ServiceFactory(name=f"Service {n}") - StatusPageFactory(service=service) - _with_status(service, Severity.OPERATIONAL, is_overall=True) - - counts = {} - with override_settings(DEBUG=True): - for size in (2, 12): - reset_queries() - APIClient().get(reverse("service-list"), {"page_size": size}) - counts[size] = len(connection.queries) - - assert counts[2] == counts[12], counts diff --git a/api/tests/test_common_api.py b/api/tests/test_common_api.py index 9859eff..32ee099 100644 --- a/api/tests/test_common_api.py +++ b/api/tests/test_common_api.py @@ -82,5 +82,5 @@ def test_a_view_that_declares_no_permissions_requires_authentication(): def test_the_catalog_stays_public_on_purpose(client): # Browsing without an account is in the spec, so these say AllowAny # rather than inheriting it. - assert client.get(reverse("service-list")).status_code == 200 + assert client.get(reverse("component-list")).status_code == 200 assert client.get(reverse("meta")).status_code == 200 diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py index 2a41c0d..9a94e0e 100644 --- a/api/tests/test_contract.py +++ b/api/tests/test_contract.py @@ -79,13 +79,6 @@ def _me_probe(): return api, reverse("me"), "email" -def _service_list_probe(): - service = ServiceFactory() - StatusPageFactory(service=service) - ComponentFactory(service=service, is_overall=True) - return APIClient(), reverse("service-list"), "id" - - def _service_detail_probe(): service = ServiceFactory() StatusPageFactory(service=service) @@ -93,27 +86,6 @@ def _service_detail_probe(): return APIClient(), reverse("service-detail", kwargs={"slug": service.slug}), "id" -def _service_components_probe(): - service = ServiceFactory() - ComponentFactory(service=service, is_overall=True) - url = reverse("service-components", kwargs={"slug": service.slug}) - return APIClient(), url, "id" - - -def _service_events_probe(): - service = ServiceFactory() - ServiceEvent.objects.create( - service=service, - external_id="probe", - kind=EventKind.INCIDENT, - title="x", - phase=IncidentPhase.DETECTED, - starts_at=timezone.now(), - ) - url = reverse("service-events", kwargs={"slug": service.slug}) - return APIClient(), url, "id" - - def _event_list_probe(): service = ServiceFactory() ServiceEvent.objects.create( @@ -189,10 +161,7 @@ def _board_components_probe(): FIELDS_PROBES = { "/meta/": _meta_probe, "/me/": _me_probe, - "/catalog/services/": _service_list_probe, "/catalog/services/{slug}/": _service_detail_probe, - "/catalog/services/{slug}/components/": _service_components_probe, - "/catalog/services/{slug}/events/": _service_events_probe, "/catalog/components/": _component_list_probe, "/catalog/components/{uuid}/": _component_detail_probe, "/dashboards/{uuid}/components/": _board_components_probe, diff --git a/api/tests/test_event_api.py b/api/tests/test_event_api.py index ddc2aca..f943a12 100644 --- a/api/tests/test_event_api.py +++ b/api/tests/test_event_api.py @@ -77,6 +77,55 @@ def test_component_narrows_the_feed(client): assert [r["title"] for r in results] == ["Wanted"] +def test_kind_narrows_the_feed(client): + # Incidents and maintenance are one collection. A screen showing + # only one of them says which. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Broke") + _event( + service, + component, + external_id="2", + title="Planned", + kind=EventKind.MAINTENANCE, + phase=MaintenancePhase.SCHEDULED, + ) + + both = client.get(reverse("event-list")).json()["results"] + incidents = client.get(reverse("event-list"), {"kind": "incident"}).json()[ + "results" + ] + assert len(both) == 2 + assert [r["title"] for r in incidents] == ["Broke"] + + +def test_the_feed_aggregate_is_by_phase_only(client): + # A severity belongs to a component. An event carries a phase, so + # a severity count here would answer about the wrong row. + aggregates = client.get(reverse("event-list")).json()["aggregates"] + assert set(aggregates) == {"total", "by_phase"} + + +def test_an_event_carries_no_component_ids(client): + # The projection runs one way. A component carries its event, and + # the reverse would put an unbounded list on every row. + service = ServiceFactory() + _event(service, ComponentFactory(service=service), external_id="1") + + row = client.get(reverse("event-list")).json()["results"][0] + assert set(row) == { + "id", + "kind", + "title", + "phase", + "starts_at", + "ends_at", + "detected_by", + "service", + } + + def test_phase_open_and_closed_draw_one_line(client): # `CLOSED_PHASES` lives in status/choices.py. A client restating # which phases are terminal is a second copy that can drift. diff --git a/api/tests/test_import.py b/api/tests/test_import.py index d282770..b590672 100644 --- a/api/tests/test_import.py +++ b/api/tests/test_import.py @@ -88,8 +88,9 @@ def test_urls_are_deduplicated_after_normalising(): @pytest.mark.django_db -def test_the_imported_service_is_identical_in_shape_to_a_listed_one(): - # If a field is missing here it belongs in the list too, not in a second shape. +def test_the_imported_service_is_identical_in_shape_to_a_read_one(): + # If a field is missing here it belongs on the service page too, + # not in a second shape. created = ( APIClient() .post( @@ -99,8 +100,8 @@ def test_the_imported_service_is_identical_in_shape_to_a_listed_one(): ) .json() ) - listed = APIClient().get(reverse("service-list")).json()["results"][0] - assert set(created) == set(listed) + url = reverse("service-detail", args=[created["slug"]]) + assert set(created) == set(APIClient().get(url).json()) @pytest.mark.django_db diff --git a/api/tests/test_security.py b/api/tests/test_security.py index 1157e4f..465ebb2 100644 --- a/api/tests/test_security.py +++ b/api/tests/test_security.py @@ -120,7 +120,7 @@ def test_reading_the_catalog_is_not_throttled_away(db): # The wide rates are the point: the catalog is public and scanned. ServiceFactory() client = APIClient() - codes = [client.get(reverse("service-list")).status_code for _ in range(30)] + codes = [client.get(reverse("component-list")).status_code for _ in range(30)] assert set(codes) == {200} diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 6f3d847..d40fc0d 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -183,39 +183,6 @@ paths: description: Removes the dashboard, its tracked components and the user. Catalog and status data are untouched — they are not personal. responses: { '204': { description: No content } } - /catalog/services/: - get: - tags: [catalog] - summary: List services - description: | - This endpoint serves Discover, the suggested lists, and the public "down right now" view. - Use parameters to select the view. - - With `q`, results are ordered by relevance. Without `q`, the default ordering is - `suggested`: `(-is_featured, severity, -watcher_count, name)`. A broken service outranks a - more popular service that is working. - security: [{}, { bearer: [] }] - parameters: - - { name: q, in: query, schema: { type: string } } - - { name: overall_component__status__severity__lte, in: query, schema: { type: integer, minimum: 0, maximum: 5 }, description: 'A service''s severity is its overall component''s, so the path runs through it.' } - - { name: overall_component__status__severity, in: query, schema: { type: integer, minimum: 0, maximum: 5 }, description: 'Maintenance chip is =4' } - - { name: overall_component__is_tracked, in: query, schema: { type: boolean }, description: 'Authenticated only. Narrower: whether the overall component specifically is tracked, which is what the ＋ on a service row toggles.' } - - { name: ordering, in: query, schema: { type: string, enum: [suggested, overall_component__status__severity, -overall_component__status__severity, name, -name, updated_at, -updated_at], default: suggested } } - - { $ref: '#/components/parameters/PageSize' } - - { $ref: '#/components/parameters/Cursor' } - - { $ref: '#/components/parameters/Fields' } - responses: - '200': - description: OK - content: - application/json: - schema: - allOf: - - $ref: '#/components/schemas/Envelope' - - type: object - properties: - aggregates: { $ref: '#/components/schemas/StatusAggregates' } - results: { type: array, items: { $ref: '#/components/schemas/Service' } } /catalog/import/: post: tags: [catalog] @@ -339,76 +306,6 @@ paths: '200': { description: OK, content: { application/json: { schema: { $ref: '#/components/schemas/Service' } } } } '404': { $ref: '#/components/responses/Error' } - /catalog/services/{slug}/components/: - get: - tags: [catalog] - summary: List a service's components - description: | - The component tree, flattened. `parent` links a child to its group. The overall component - is first. `aggregates` gives the tab labels. - security: [{}, { bearer: [] }] - parameters: - - { $ref: '#/components/parameters/Slug' } - - { name: is_tracked, in: query, schema: { type: boolean }, description: Drives Showing All / Tracked / Untracked } - - { name: status__severity__lte, in: query, schema: { type: integer } } - - { name: status__severity, in: query, schema: { type: integer } } - - { name: ordering, in: query, schema: { type: string, enum: [status_page_order, status__severity, -status__severity, name, -name], default: status_page_order }, description: '`status_page_order` keeps the order the status page lists them in.' } - - { $ref: '#/components/parameters/PageSize' } - - { $ref: '#/components/parameters/Cursor' } - - { $ref: '#/components/parameters/Fields' } - responses: - '200': - description: OK - content: - application/json: - schema: - allOf: - - $ref: '#/components/schemas/Envelope' - - type: object - properties: - aggregates: { $ref: '#/components/schemas/StatusAggregates' } - results: { type: array, items: { $ref: '#/components/schemas/Component' } } - - /catalog/services/{slug}/events/: - get: - tags: [catalog] - summary: List a service's events - description: | - Incidents and maintenance are one collection. Filter with `kind`. - - The Incidents tab passes `kind=incident&ends_at__isnull=true`. The Maintenance tab passes - `kind=maintenance` with a window filter. - security: [{}, { bearer: [] }] - parameters: - - { $ref: '#/components/parameters/Slug' } - - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] }, description: Omit for both. } - - { name: phase, in: query, schema: { type: string }, description: 'Valid values depend on `kind`. See `/meta/` `enums.event_phase`.' } - - { name: phase__in, in: query, schema: { type: string }, description: Comma-separated. } - - { name: ends_at__isnull, in: query, schema: { type: boolean }, description: '`true` is the open set — unresolved, or a window still to close.' } - - { name: starts_at__gte, in: query, schema: { type: string, format: date-time } } - - { name: starts_at__lte, in: query, schema: { type: string, format: date-time }, description: 'With `ends_at__gte`, answers "what happens between now and Thursday".' } - - { name: ends_at__gte, in: query, schema: { type: string, format: date-time } } - # The M2M filter stays although no field exposes it. django-filter generates it free, - # and narrowing events to one component is a query a screen may want. The field was - # different: a prefetch and bytes on every row that nothing rendered. - - { name: affected_components, in: query, schema: { type: string, format: uuid }, description: Events naming this component. } - - { name: ordering, in: query, schema: { type: string, enum: [-starts_at, starts_at, ends_at, -ends_at], default: -starts_at } } - - { $ref: '#/components/parameters/PageSize' } - - { $ref: '#/components/parameters/Cursor' } - - { $ref: '#/components/parameters/Fields' } - responses: - '200': - description: OK - content: - application/json: - schema: - allOf: - - $ref: '#/components/schemas/Envelope' - - type: object - properties: - aggregates: { $ref: '#/components/schemas/EventAggregates' } - results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } - /events/: get: tags: [status] @@ -485,27 +382,15 @@ paths: description: Filtering, sorting and pagination are server-side without exception — nothing assumes a board is small. parameters: - { $ref: '#/components/parameters/Uuid' } - - name: event - in: query - schema: { type: string, enum: [incident, maintenance] } - description: | - The Incidents and Maintenance tabs. Keep a row if the component has an open - `ServiceEvent` of this kind. Omit the parameter for the All tab. - - Declared, not generated. `ServiceEvent` and `ServiceComponent` are many-to-many, and - two generated parameters would apply as two `.filter()` calls. On a many-to-many each - call joins the relation again, so `events__kind` and `events__phase` can be satisfied - by two different events: a component with one closed incident and one running - maintenance would match `event=incident`. One `.filter()` holding both conditions - binds them to the same row. - - { name: status__severity__lte, in: query, schema: { type: integer }, description: 'Severity, not events. It works inside any tab — `event=incident&status__severity__lte=3` is written-up incidents that are also down now.' } + - { name: status__severity__lte, in: query, schema: { type: integer } } - { name: status__severity, in: query, schema: { type: integer } } + - { name: status__severity__in, in: query, schema: { type: string }, description: 'Comma separated, for a Severity filter offering every value at once.' } - name: ordering in: query schema: { type: string, enum: [status__severity, -status__severity, name, -name, updated_at, -updated_at, next_transition], default: status__severity } description: | - `next_transition` sorts the Maintenance tab. It uses `ends_at` for a running window - and `starts_at` for one that has not started. Running windows sort first. This + `next_transition` sorts by the next maintenance step. It uses `ends_at` for a running + window and `starts_at` for one that has not started. Running windows sort first. This ordering is declared, not generated. - { $ref: '#/components/parameters/PageSize' } - { $ref: '#/components/parameters/Cursor' } @@ -695,16 +580,6 @@ components: additionalProperties: { type: integer } description: | Keys are severities 0–5. Sorting and the dot read from this. No tab is a severity. - by_event_kind: - type: object - additionalProperties: { type: integer } - description: | - `{"incident": 3, "maintenance": 1}` — how many rows each value of `event` returns. - These are the Incidents and Maintenance tab counts. - - Neither can be read off `by_severity`. A maintenance window three days out leaves - the component at severity 5, and a provider can open an incident on components it - still reports as operational. next_refresh_at: type: string format: date-time @@ -888,10 +763,6 @@ components: A value above 0 means the service is on your board. This differs from `overall_component.is_tracked`, which covers the overall component only. - description: - type: string - nullable: true - description: The provider's own summary of the service. The About tab shows it. in_catalog_since: type: string format: date @@ -968,7 +839,7 @@ components: The full list is not on the component. History is unbounded, and a row renders from the soonest plus this count. Read the list from - `GET /catalog/services/{slug}/events/?kind=maintenance` when a screen needs it. + `GET /events/?component={uuid}&kind=maintenance` when a screen needs it. active_incident: allOf: [{ $ref: '#/components/schemas/EventRef' }] description: | diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index f9f7a3c..fbf0c9e 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -28,7 +28,7 @@ current state, read the provider's incident log, and browse a public view withou | Community issue reporting, comments | Sub-project #3. The Figma set already contains `status dot - user reported`, a split-circle variant, so the dot component should accept a shape from day one. | | Notifications | Sub-project #2. The Figma set contains a `Toggle Notifications` button; nothing in v1 uses it. | | Uptime history charts | `ComponentStatus` accumulates from first poll, so any chart is empty on day one. Ships when there is data. | -| Service categories | Cut deliberately. `is_featured` plus `watcher_count` orders the suggested list instead. | +| Service categories | Cut deliberately. `is_featured` plus `watcher_count` orders the suggested list instead. Both sit on `ServiceComponent`. | ### Non-goals @@ -126,11 +126,8 @@ erDiagram uuid id PK string slug UK string name - text description "from the provider's page" url logo url homepage_url - bool is_featured "first key of the suggested sort" - int watcher_count "distinct users" } StatusPage { uuid id PK @@ -258,20 +255,16 @@ is a different concept from a permission role. ### catalog -- `Service` — slug, name, `description`, logo, homepage URL, `is_featured`, `watcher_count` - - `description` and `logo` come from the provider's page and are refreshed on every poll, so a - rename or a rebrand upstream does not leave a stale entry in the catalogue. The service screen's - About tab renders the description. +- `Service` — slug, name, logo, homepage URL - `is_featured` is a boolean ticked in admin for services worth surfacing regardless of usage. It - is the first key of the suggestion sort, and on day one — every `watcher_count` zero — it is the - whole list. + `logo` comes from the provider's page and is refreshed on every poll, so a rebrand upstream does + not leave a stale entry in the catalogue. - Two earlier fields are gone. `added_by` duplicated `BaseModel.created_by`, which every model + Three earlier fields are gone. `added_by` duplicated `BaseModel.created_by`, which every model already carries. `is_curated` had no consumer: nothing sorted, filtered or rendered by it. - "Seeded by us rather than pasted by someone" is `created_by IS NULL`, and "worth surfacing" is - `is_featured`. + "Seeded by us rather than pasted by someone" is `created_by IS NULL`. `watcher_count` was a + column a signal kept true, and four write paths never reached the signal. It is counted when it + is read instead. - `StatusPage` — **its own model**, `OneToOneField` to `Service`. `url` (normalised, **unique**), `provider`, `api_url_override` (blank unless needed). What and where — nothing about polling. - `Poller` — `OneToOneField` to **`Service`**, created with it. The thing that reads a service: @@ -348,27 +341,23 @@ is a different concept from a permission role. parents costs nothing to keep in step. Custom URLs dedupe on `StatusPage.url`, so two users pasting the same status page share one -`Service` and one poll. A popular pasted entry is surfaced by ticking `is_featured` in admin. +`Service` and one poll. A popular pasted entry is surfaced by ticking `is_featured` on its overall +component in admin. -**Suggestion ordering** is `(-is_featured, -watcher_count, name)`. +**Suggestion ordering** is `(-is_featured, severity, -watcher_count, name)`, over components. -**`watcher_count` is distinct users, not items.** Someone tracking five Twilio components is one -watcher, not five: +**`watcher_count` is distinct users, not items.** Someone tracking one component from two +boards is one watcher, not two: ```sql SELECT COUNT(DISTINCT d.owner_id) FROM dashboards_dashboarditem i JOIN dashboards_dashboard d ON d.id = i.dashboard_id -WHERE i.service_id = %s +WHERE i.component_id = %s ``` -It is recomputed for the one affected service on `DashboardItem` create and delete — a single -indexed aggregate, not a scan — so it is exact rather than drifting the way naive -increment/decrement would when someone adds a second component for a service they already track. -Deleting an account removes its items, which fires the same path. - -A nightly reconciliation task recomputes every service as a safety net. It orders a suggestion -list, so a few hours of staleness costs nothing if a signal is ever missed. +It is counted when the list is read, so it is exact. There is no column and no signal to +keep one true. On day one every `watcher_count` is zero, so the list is your featured picks followed by everything else alphabetically. As real usage arrives the tail self-orders and you stop curating. @@ -652,7 +641,7 @@ Each successful poll: 2. **Marks vanished components `archived_at`** rather than deleting them. Someone may be tracking one, and deleting the row would silently remove it from their board. An archived component still renders, reads `unknown`, and says the provider no longer publishes it. -3. **Refreshes the service's own metadata** — name, description, logo — for seeded and pasted +3. **Refreshes the service's own metadata** — name, homepage URL, logo — for seeded and pasted entries alike, so a rename upstream does not leave a stale name on someone's board. Component identity is the provider's `external_id`, never the display name. Names change; ids do From 9dd2934c5dee92cf82fe7097f5b9eafa6fbca242 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 19:11:07 -0700 Subject: [PATCH 18/94] feat: admin features a service from the service page is_featured moved to the overall component, and an admin asks to feature a service rather than a component. The component list shows watchers and gains a bulk action to feature or unfeature the overall components in a selection, and service requests are ordered by how often they are asked for. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 98 ++++++++++++++++++++++++++++++- api/tests/test_admin.py | 127 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 2 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index 2f1d46a..d9a5391 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -18,8 +18,8 @@ from unfold.enums import ActionVariant from unfold.forms import BaseDialogForm -from catalog.models import Service, ServiceComponent, StatusPage -from catalog.queries import OVERALL_SEVERITY, is_tracked +from catalog.models import Service, ServiceComponent, ServiceRequest, StatusPage +from catalog.queries import COMPONENT_WATCHER_COUNT, OVERALL_SEVERITY, is_tracked from common.admin import ( SEVERITY_VARIANTS, BaseModelAdmin, @@ -304,6 +304,35 @@ def queryset(self, request, queryset): ) +class OverallComponentInline(StackedInline): + """Featuring lives on the rollup, and an admin asks about the service. + + `is_featured` is the first key of the suggested sort. Ticking it on + a service's overall component is what surfaces that service, so + that is the question this answers. + + `ServiceComponentInline` also edits `ServiceComponent`, through the + same FK. `ModelAdmin._create_formsets` auto-suffixes a repeated + prefix, so the two never collide even though both derive the same + default one. + """ + + model = ServiceComponent + fields = ["is_featured"] + extra = 0 + max_num = 1 + can_delete = False + verbose_name = "Featuring" + verbose_name_plural = "Featuring" + + def get_queryset(self, request): + return super().get_queryset(request).filter(is_overall=True) + + def has_add_permission(self, request, obj): + # Reconcile makes the rollup. An admin ticks its flag. + return False + + @admin.register(Service) class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): list_display = [ @@ -340,6 +369,7 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): # What the service is, then what is happening to it. Then how it # is configured, then the log of us reading it. inlines = [ + OverallComponentInline, ServiceComponentInline, ServiceEventInline, StatusPageInline, @@ -495,6 +525,10 @@ class ServiceComponentAdmin( "is_overall", "display_archived", "display_related", + "watchers", + # Only the overall component's flag surfaces its service. The + # flag is real on any row, but only that one row does anything. + "is_featured", ] search_fields = [ "name", @@ -514,6 +548,7 @@ class ServiceComponentAdmin( "is_archived", ("archived_at", RangeDateTimeFilter), ("created_at", RangeDateTimeFilter), + "is_featured", ] autocomplete_fields = ["service", "parent"] ordering = ["service__name", "status_page_order"] @@ -541,6 +576,11 @@ class ServiceComponentAdmin( audit_section(), ] inlines = [ComponentStatusInline] + # Two, not a toggle. A selection can mix already-featured and not, + # and a toggle would leave an admin unable to predict the result. + # `PollerAdmin.toggle_pause` can be one action because it only ever + # acts on a single object, which has one state to flip. + actions = ["feature_selected", "unfeature_selected"] def get_form(self, request, obj=None, **kwargs): # The parent picker needs to know which service is asking. @@ -572,6 +612,7 @@ def get_queryset(self, request): severity_now=CURRENT_SEVERITY, status_count=related_count(ComponentStatus.objects, "component"), event_count=related_count(ServiceEvent.objects, "affected_components"), + watcher_count=COMPONENT_WATCHER_COUNT, ) ) @@ -622,3 +663,56 @@ def display_related(self, obj): @display(description=_("Status"), label=SEVERITY_VARIANTS, ordering="severity_now") def display_severity(self, obj): return severity_label(obj.severity_now) + + @display(description=_("Watchers"), ordering="watcher_count") + def watchers(self, obj): + return obj.watcher_count + + def _skip_note(self, queryset): + """How many of the selection the action would not touch, and why. + + `suggested` sorts on `-is_featured` first, over every component, + rollups included. Featuring a leaf would win Discover outright, + which nothing intends. Only the overall component may hold the + flag, so a silent skip would look like nothing happened. + """ + skipped = queryset.exclude(is_overall=True).count() + if not skipped: + return "" + return " " + _( + "%(skipped)d selected component(s) skipped: not an overall component." + ) % {"skipped": skipped} + + @admin.action(description=_("Feature selected components")) + def feature_selected(self, request, queryset): + count = 0 + for component in queryset.filter(is_overall=True, is_featured=False): + component.is_featured = True + component.save(update_fields=["is_featured"]) + count += 1 + message = _("Featured %(count)d component(s).") % {"count": count} + self.message_user(request, message + self._skip_note(queryset)) + + @admin.action(description=_("Unfeature selected components")) + def unfeature_selected(self, request, queryset): + count = 0 + for component in queryset.filter(is_overall=True, is_featured=True): + component.is_featured = False + component.save(update_fields=["is_featured"]) + count += 1 + message = _("Unfeatured %(count)d component(s).") % {"count": count} + self.message_user(request, message + self._skip_note(queryset)) + + +@admin.register(ServiceRequest) +class ServiceRequestAdmin(ModelAdmin): + """What the catalog is missing, most asked for first.""" + + list_display = ["url", "request_count", "last_requested_at", "created_by"] + ordering = ["-request_count", "-last_requested_at"] + search_fields = ["url"] + readonly_fields = ["url", "request_count", "last_requested_at"] + + def has_add_permission(self, request): + # A row arrives from the app, never from here. + return False diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index c3f9193..f0a823e 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -625,6 +625,133 @@ def test_a_person_cannot_open_somebody_elses_board(): mine.full_clean() +@pytest.mark.django_db +def test_the_service_admin_features_through_the_overall_component(admin_client, db): + # is_featured moved to the component. An admin asks "feature this + # service", so the service page has to answer it. + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + response = admin_client.get( + reverse("admin:catalog_service_change", args=[service.pk]) + ) + assert response.status_code == 200 + assert b"is_featured" in response.content + + +@pytest.mark.django_db +def test_the_component_list_shows_watchers(admin_client, db): + # The count is an annotation. A column that forgets to annotate it + # raises rather than showing a wrong number. A column with no + # ordering field cannot be sorted at all. + from catalog.models import ServiceComponent + from tests.factories import ComponentFactory, track + + ComponentFactory(name="Quiet") + loud = ComponentFactory(name="Loud") + track(loud) + track(loud) + + model_admin = admin.site._registry[ServiceComponent] + index = model_admin.list_display.index("watchers") + 1 + response = admin_client.get( + reverse("admin:catalog_servicecomponent_changelist"), {"o": f"-{index}"} + ) + assert response.status_code == 200 + assert b">2<" in response.content + assert response.content.index(b"Loud") < response.content.index(b"Quiet") + + +@pytest.mark.django_db +def test_service_requests_are_listed_by_demand(admin_client, db): + # The list is the demand signal for what the catalog is missing, + # so the most-asked-for URL is the first row. + from catalog.models import ServiceRequest + + ServiceRequest.objects.create(url="https://a.example", request_count=1) + ServiceRequest.objects.create(url="https://b.example", request_count=9) + response = admin_client.get(reverse("admin:catalog_servicerequest_changelist")) + assert response.status_code == 200 + assert response.content.index(b"b.example") < response.content.index(b"a.example") + + +@pytest.mark.django_db +def test_the_feature_action_flips_the_flag(admin_client, db): + # Selecting a component and running the action is how the spec's + # "bulk action" is meant to be used. A missing action leaves an + # admin with no way to feature many components at once. + from tests.factories import ComponentFactory + + component = ComponentFactory(is_featured=False, is_overall=True) + admin_client.post( + reverse("admin:catalog_servicecomponent_changelist"), + {"action": "feature_selected", "_selected_action": [str(component.pk)]}, + ) + component.refresh_from_db() + assert component.is_featured is True + + +@pytest.mark.django_db +def test_the_unfeature_action_flips_it_back(admin_client, db): + # The reverse has to exist too. Otherwise featuring by mistake is + # permanent, short of editing the row through the service's inline. + from tests.factories import ComponentFactory + + component = ComponentFactory(is_featured=True, is_overall=True) + admin_client.post( + reverse("admin:catalog_servicecomponent_changelist"), + {"action": "unfeature_selected", "_selected_action": [str(component.pk)]}, + ) + component.refresh_from_db() + assert component.is_featured is False + + +@pytest.mark.django_db +def test_the_feature_action_skips_a_non_overall_component(admin_client, db): + # `suggested` sorts every component on `-is_featured` first, rollups + # included. Featuring a leaf would win Discover outright. Nothing in + # the design intends that, so the action must refuse it and say why. + from tests.factories import ComponentFactory + + leaf = ComponentFactory(is_featured=False, is_overall=False) + response = admin_client.post( + reverse("admin:catalog_servicecomponent_changelist"), + {"action": "feature_selected", "_selected_action": [str(leaf.pk)]}, + follow=True, + ) + leaf.refresh_from_db() + assert leaf.is_featured is False + assert b"skipped" in response.content + + +@pytest.mark.django_db +def test_a_mixed_selection_features_the_overall_one_and_skips_the_leaf( + admin_client, db +): + # A selection an admin makes by hand is not guaranteed to be clean. + # The overall row still gets featured; the leaf does not, silently + # or otherwise. + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory() + overall = ComponentFactory(service=service, is_overall=True, is_featured=False) + leaf = ComponentFactory(service=service, is_overall=False, is_featured=False) + response = admin_client.post( + reverse("admin:catalog_servicecomponent_changelist"), + { + "action": "feature_selected", + "_selected_action": [str(overall.pk), str(leaf.pk)], + }, + follow=True, + ) + overall.refresh_from_db() + leaf.refresh_from_db() + assert overall.is_featured is True + assert leaf.is_featured is False + assert b"skipped" in response.content + + @pytest.mark.django_db def test_the_polling_schedule_page_opens(staff_client): """Readonly is not free: Django drops a readonly field from the form. From 6842cc3cab9aacbc4f158d65549df4d12f7ac5ba Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 19:51:31 -0700 Subject: [PATCH 19/94] test: move the prose-checker test beside the script it tests api/tests/test_prose_checker.py loaded bin/check_prose.py by path, but the API image builds from api/ alone. `just check` proves the image, and bin/ is absent there, so the test failed only in the image: 10 errors, every one a FileNotFoundError. AGENTS.md rules out copying bin/ into the API image: one image per component, built from that component's own directory. It also names the fix: a stdlib-only script is the exception to running gated checks in a container, because a container costs more than it saves. check_prose.py is stdlib-only, so its test moves to bin/tests/test_check_prose.py and runs with `python3 -m unittest`, needing no venv and no Django. `just check` now runs it on the host as `test-bin`, beside the existing check_prose.py invocation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_prose_checker.py | 66 --------------------------------- bin/tests/test_check_prose.py | 62 +++++++++++++++++++++++++++++++ justfile | 5 +++ 3 files changed, 67 insertions(+), 66 deletions(-) delete mode 100644 api/tests/test_prose_checker.py create mode 100644 bin/tests/test_check_prose.py diff --git a/api/tests/test_prose_checker.py b/api/tests/test_prose_checker.py deleted file mode 100644 index 7deda20..0000000 --- a/api/tests/test_prose_checker.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Behaviour tests for `bin/check_prose.py`. - -`bin/` has no test runner of its own, and the only one in this repo is -this Django suite. `check_prose.py` needs no Django, so it is loaded -by path rather than inventing a second runner for one script. -""" - -import importlib.util -from pathlib import Path - -CHECK_PROSE = Path(__file__).resolve().parents[2] / "bin" / "check_prose.py" - - -def _load_check_prose(): - spec = importlib.util.spec_from_file_location("check_prose", CHECK_PROSE) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -check_prose = _load_check_prose() - - -def test_a_multi_word_code_span_counts_as_one_word(): - # The old filter dropped only the opening token. A five-word span - # leaked four words into the count instead of naming one. - with_span = check_prose.words("Reject the row unless `external_id IS NULL` matches") - without_span = check_prose.words("Reject the row unless matches") - assert with_span == without_span + 1 - - -def test_a_single_token_code_span_still_counts_as_one_word(): - # The old filter dropped a lone backtick-prefixed token outright, - # so a sentence naming one field was undercounted by one. - assert check_prose.words( - "Reject the row unless `external_id` matches" - ) == check_prose.words("Reject the row unless `external_id IS NULL` matches") - - -def test_a_code_span_followed_by_a_comma_counts_only_the_span(): - # A padded substitution once tore the comma into its own token, - # counting a span and its punctuation as two words. - assert check_prose.words("The `foo`, which is set, wins") == 6 - - -def test_a_code_span_followed_by_a_full_stop_counts_only_the_span(): - # Same bug at a sentence end. The period must not become a free - # word once the span collapses. - assert check_prose.words("Set by `foo`. Then read") == 5 - - -def test_an_unmatched_backtick_does_not_raise(): - # sentences() can in principle hand words() a fragment holding half - # a span. That must fall back to counting, not crash the checker. - assert check_prose.words("A stray ` mark stays here") == 5 - - -def test_a_twenty_word_sentence_is_not_flagged(): - sentence = " ".join(["word"] * 20) - assert not check_prose.prose_problems([(sentence, 1)]) - - -def test_a_twenty_one_word_sentence_is_flagged(): - sentence = " ".join(["word"] * 21) - problems = check_prose.prose_problems([(sentence, 1)]) - assert [rule for _line, rule, _detail in problems] == ["long"] diff --git a/bin/tests/test_check_prose.py b/bin/tests/test_check_prose.py new file mode 100644 index 0000000..702da9b --- /dev/null +++ b/bin/tests/test_check_prose.py @@ -0,0 +1,62 @@ +"""Behaviour tests for `bin/check_prose.py`. + +`bin/` has no dependency manager and no Django. These tests use only the +standard library, so they run on a machine that has done nothing but +`just init`. Run them with `python3 -m unittest discover -s bin/tests`. +""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import check_prose + + +class WordsTests(unittest.TestCase): + def test_a_multi_word_code_span_counts_as_one_word(self): + # The old filter dropped only the opening token. A five-word span + # leaked four words into the count instead of naming one. + with_span = check_prose.words( + "Reject the row unless `external_id IS NULL` matches" + ) + without_span = check_prose.words("Reject the row unless matches") + self.assertEqual(with_span, without_span + 1) + + def test_a_single_token_code_span_still_counts_as_one_word(self): + # The old filter dropped a lone backtick-prefixed token outright, + # so a sentence naming one field was undercounted by one. + self.assertEqual( + check_prose.words("Reject the row unless `external_id` matches"), + check_prose.words("Reject the row unless `external_id IS NULL` matches"), + ) + + def test_a_code_span_followed_by_a_comma_counts_only_the_span(self): + # A padded substitution once tore the comma into its own token, + # counting a span and its punctuation as two words. + self.assertEqual(check_prose.words("The `foo`, which is set, wins"), 6) + + def test_a_code_span_followed_by_a_full_stop_counts_only_the_span(self): + # Same bug at a sentence end. The period must not become a free + # word once the span collapses. + self.assertEqual(check_prose.words("Set by `foo`. Then read"), 5) + + def test_an_unmatched_backtick_does_not_raise(self): + # sentences() can in principle hand words() a fragment holding half + # a span. That must fall back to counting, not crash the checker. + self.assertEqual(check_prose.words("A stray ` mark stays here"), 5) + + +class ProseProblemsTests(unittest.TestCase): + def test_a_twenty_word_sentence_is_not_flagged(self): + sentence = " ".join(["word"] * 20) + self.assertFalse(check_prose.prose_problems([(sentence, 1)])) + + def test_a_twenty_one_word_sentence_is_flagged(self): + sentence = " ".join(["word"] * 21) + problems = check_prose.prose_problems([(sentence, 1)]) + self.assertEqual([rule for _line, rule, _detail in problems], ["long"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/justfile b/justfile index 7e66b0e..167eae8 100644 --- a/justfile +++ b/justfile @@ -95,6 +95,10 @@ lint: prose *args: @python3 bin/check_prose.py {{args}} +# Test bin/ scripts. Stdlib only, so it needs no venv and no Django. +test-bin: + @python3 -m unittest discover -s bin/tests -v + # Slower than `just test`, and the answer that counts: it proves the # image rather than the host. Run it before opening a pull request. @@ -104,6 +108,7 @@ check: image docker compose -f docker-compose.test.yml run --rm ruff docker compose -f docker-compose.test.yml run --rm docs @python3 bin/check_prose.py + @just test-bin docker compose -f docker-compose.test.yml run --rm --entrypoint sh pytest \ -c 'python manage.py makemigrations --check --dry-run' docker compose -f docker-compose.test.yml down -v From ff00a44af0641ccb921f882977612878907168a5 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 21:33:34 -0700 Subject: [PATCH 20/94] feat: the event admin says who found an outage and who wrote an update `detected_by` exists to answer "did we find this first". A claim fills `external_id` in, so nothing else can answer it. The column was on no changelist, no filter and no form, so the admin could not ask. `source` has the same problem on the timeline. A claimed event holds our updates and the provider's, interleaved. Read without the source they look like one author. Both are read only, because `PollerWrittenAdmin` already refuses the three permissions on these tables. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/status/admin.py | 34 ++++++++++-- api/tests/test_admin.py | 120 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 6 deletions(-) diff --git a/api/status/admin.py b/api/status/admin.py index c175e4f..b76b21e 100644 --- a/api/status/admin.py +++ b/api/status/admin.py @@ -131,10 +131,11 @@ def display_service(self, obj): class EventUpdateInline(TabularInline): - """The provider's update log, read only. + """The event's update log, read only. - These are the provider's words, written by a poll. Editing them here - would make the admin disagree with the status page it mirrors. + A poll writes these. Most are the provider's words. A claimed event + also holds ours. Editing any of them here would make the admin + disagree with the status page it mirrors. """ model = EventUpdate @@ -145,10 +146,16 @@ class EventUpdateInline(TabularInline): # Read only, but still worth opening: the row is a summary. show_change_link = True per_page = 10 - fields = ["phase", "body", "posted_at"] + # A claimed event interleaves our updates with the provider's. + # Without the source the timeline reads as one author. + fields = ["phase", "display_source", "body", "posted_at"] readonly_fields = fields ordering = ["-posted_at"] + @display(description=_("Source")) + def display_source(self, obj): + return obj.get_source_display() + def has_add_permission(self, request, obj=None): return False @@ -161,6 +168,7 @@ class ServiceEventAdmin(PollerWrittenAdmin, ModelAdmin): "display_when", "display_kind", "display_phase", + "display_detected_by", "display_related", ] date_hierarchy = "starts_at" @@ -172,6 +180,7 @@ class ServiceEventAdmin(PollerWrittenAdmin, ModelAdmin): ("affected_components", AutocompleteSelectFilter), ("poll_run", AutocompleteSelectFilter), ("kind", ChoicesDropdownFilter), + ("detected_by", ChoicesDropdownFilter), PhaseFilter, ("starts_at", RangeDateTimeFilter), ("ends_at", RangeDateTimeFilter), @@ -254,7 +263,7 @@ def get_fieldsets(self, request, obj=None): ] }, ), - (_("Classification"), {"fields": ["kind", "phase"]}), + (_("Classification"), {"fields": ["kind", "phase", "detected_by"]}), (_("Span"), {"fields": ["starts_at", "ends_at"]}), audit_section(), ] @@ -321,6 +330,12 @@ def display_affected_components(self, obj): def display_kind(self, obj): return EventKind(obj.kind).label + @display(description=_("Detected by"), ordering="detected_by") + def display_detected_by(self, obj): + # `external_id IS NULL` cannot answer this. A claim fills the + # column in, which destroys the fact that we found it first. + return obj.get_detected_by_display() + @display(description=_("Phase"), label=True, ordering="phase") def display_phase(self, obj): # A closed phase means the event is over. @@ -338,6 +353,7 @@ class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): "display_event", "display_service", "phase", + "display_source", "posted_at", ] display_update = record_column(_("Update")) @@ -352,14 +368,20 @@ class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): list_filter = [ ("event__service", AutocompleteSelectFilter), ("event", AutocompleteSelectFilter), + ("source", ChoicesDropdownFilter), ("posted_at", RangeDateTimeFilter), ("created_at", RangeDateTimeFilter), ] fieldsets = [ - (None, {"fields": ["event", "phase", "posted_at", "body"]}), + (None, {"fields": ["event", "phase", "source", "posted_at", "body"]}), audit_section(), ] + @display(description=_("Source"), ordering="source") + def display_source(self, obj): + # Who wrote the post. A claimed event holds both, one per update. + return obj.get_source_display() + @display(description=_("Service"), ordering="event__service__name") def display_service(self, obj): return change_link(obj.event.service) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index f0a823e..98701dd 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -769,3 +769,123 @@ def test_the_polling_schedule_page_opens(staff_client): url = reverse("admin:django_celery_beat_periodictask_change", args=[task.pk]) assert staff_client.get(url).status_code == 200 + + +@pytest.fixture +def claimed_event(): + """An outage we found first, later claimed by the provider. + + Both updates hang off one event. That interleaving is what the + claim mechanism produces, and what the admin has to tell apart. + """ + from status.choices import EventKind, EventSource, IncidentPhase + from status.models import EventUpdate, ServiceEvent + from tests.factories import ServiceFactory + + service = ServiceFactory(name="Twilio") + ours = ServiceEvent.objects.create( + service=service, + kind=EventKind.INCIDENT, + title="We saw it first", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + detected_by=EventSource.SYSTEM, + ) + theirs = ServiceEvent.objects.create( + service=service, + external_id="their-1", + kind=EventKind.INCIDENT, + title="They saw it first", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + detected_by=EventSource.PROVIDER, + ) + EventUpdate.objects.create( + event=ours, + phase=IncidentPhase.DETECTED, + body="Our detection", + posted_at=timezone.now(), + source=EventSource.SYSTEM, + ) + EventUpdate.objects.create( + event=ours, + phase=IncidentPhase.INVESTIGATING, + body="Their first post", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) + return ours, theirs + + +@pytest.mark.django_db +def test_the_event_list_says_who_found_the_outage(staff_client, claimed_event): + # `detected_by` answers "did we find this first". It was on no + # changelist, so the table could not be read for it. + body = staff_client.get( + reverse("admin:status_serviceevent_changelist") + ).content.decode() + assert "field-display_detected_by" in body + assert ">Statusboard" in body + + +@pytest.mark.django_db +def test_events_can_be_filtered_by_who_detected_them(staff_client, claimed_event): + # Two halves. The option has to be offered, and picking it has to + # narrow. Django allows a local-field lookup whether or not a filter + # declares it, so the queryset alone proves nothing. + from status.choices import EventSource + + ours, _theirs = claimed_event + url = reverse("admin:status_serviceevent_changelist") + assert 'id="id_detected_by__exact"' in staff_client.get(url).content.decode() + + response = staff_client.get(url, {"detected_by__exact": EventSource.SYSTEM}) + assert list(response.context["cl"].queryset) == [ours] + + +@pytest.mark.django_db +def test_the_event_page_says_who_found_the_outage(staff_client, claimed_event): + # A claim fills `external_id` in, so the record alone cannot say who + # opened the event. Only this field can. + ours, _theirs = claimed_event + url = reverse("admin:status_serviceevent_change", args=[ours.pk]) + body = staff_client.get(url).content.decode() + assert "Detected by" in body + assert ">Statusboard" in body + + +@pytest.mark.django_db +def test_an_events_timeline_says_who_wrote_each_update(staff_client, claimed_event): + # We opened this event, so the only Provider on the page is an + # update the provider wrote. Without the source the timeline reads + # as one author. + ours, _theirs = claimed_event + url = reverse("admin:status_serviceevent_change", args=[ours.pk]) + assert ">Provider" in staff_client.get(url).content.decode() + + +@pytest.mark.django_db +def test_the_update_list_says_who_wrote_each_post(staff_client, claimed_event): + # The same question on the updates table, where a person arrives + # from a filtered link rather than from the event. + body = staff_client.get( + reverse("admin:status_eventupdate_changelist") + ).content.decode() + assert "field-display_source" in body + assert ">Statusboard" in body + assert ">Provider" in body + + +@pytest.mark.django_db +def test_updates_can_be_filtered_by_who_wrote_them(staff_client, claimed_event): + # Offered and narrowing, for the same reason as the event filter. + from status.choices import EventSource + from status.models import EventUpdate + + url = reverse("admin:status_eventupdate_changelist") + assert 'id="id_source__exact"' in staff_client.get(url).content.decode() + + response = staff_client.get(url, {"source__exact": EventSource.SYSTEM}) + assert list(response.context["cl"].queryset) == [ + EventUpdate.objects.get(source=EventSource.SYSTEM) + ] From 94334cb13f4f711386381de40cda30d9d078e810 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 21:34:52 -0700 Subject: [PATCH 21/94] fix: a component saved in the admin rebuilds its derived columns `ancestor_ids` and `search_document` had one writer, reconcile. The admin is the only other one: `parent` and `name` are editable on the component form. A component added or reparented there held no ancestry and no search vector. It was invisible to `?ancestor=`, counted by no `descendant_count`, and unfindable by `?q=`. A poll repairs a tracked service. An untracked or paused one is never selected, so the gap stayed open. `save_model` and not `ServiceComponent.save`, because reconcile writes these rows with `bulk_update` and would pay the pass per row. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 19 +++++++++++++++ api/tests/test_admin.py | 51 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index d9a5391..631d409 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -35,6 +35,7 @@ from common.queries import related_count from polling.importer import import_from_url from polling.models import Poller, PollRun +from polling.reconcile import rebuild_ancestry, rebuild_search from status.choices import EventKind, Severity from status.models import ComponentStatus, ServiceEvent from status.queries import CURRENT_SEVERITY @@ -582,6 +583,24 @@ class ServiceComponentAdmin( # acts on a single object, which has one state to flip. actions = ["feature_selected", "unfeature_selected"] + def save_model(self, request, obj, form, change): + """Keep the derived columns true after an edit by hand. + + `rebuild_ancestry` and `rebuild_search` run from reconcile only. + A component added or reparented here held no ancestry and no + search vector. It was then invisible to `?ancestor=`, counted by + no `descendant_count`, and unfindable by `?q=`. + + A poll repairs a tracked service. An untracked or paused one is + never selected, so the staleness would be permanent. + + Here and not on `ServiceComponent.save`, because reconcile writes + these rows with `bulk_update` and would pay the pass per row. + """ + super().save_model(request, obj, form, change) + rebuild_ancestry(obj.service) + rebuild_search(obj.service) + def get_form(self, request, obj=None, **kwargs): # The parent picker needs to know which service is asking. request._editing = obj diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 98701dd..d034e43 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -889,3 +889,54 @@ def test_updates_can_be_filtered_by_who_wrote_them(staff_client, claimed_event): assert list(response.context["cl"].queryset) == [ EventUpdate.objects.get(source=EventSource.SYSTEM) ] + + +def component_form_data(staff_client, component, **fields): + """The component change form, with a management form per inline. + + The prefixes are read off the rendered page. A browser posts what + the page asks for, and a missing management form is a 200 that + saved nothing. + """ + import re + + url = reverse("admin:catalog_servicecomponent_change", args=[component.pk]) + body = staff_client.get(url).content.decode() + data = { + "service": str(component.service_id), + "name": component.name, + "external_id": component.external_id, + "is_archived": "", + "parent": "", + "status_page_order": str(component.status_page_order), + "is_overall": "", + **fields, + } + for prefix in set(re.findall(r'name="([\w-]+)-TOTAL_FORMS"', body)): + for key in ("TOTAL_FORMS", "INITIAL_FORMS", "MIN_NUM_FORMS", "MAX_NUM_FORMS"): + data[f"{prefix}-{key}"] = "0" + return url, data + + +@pytest.mark.django_db +def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): + """The admin is the only writer outside reconcile. + + `rebuild_ancestry` and `rebuild_search` run on a poll. A component + reparented here held no ancestry and no search vector. It was then + invisible to `?ancestor=` and unfindable by `?q=`. An untracked + service is never polled, so nothing would repair it. + """ + from catalog.models import ServiceComponent + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory(name="Twilio") + parent = ComponentFactory(service=service, name="Programmable Messaging") + child = ComponentFactory(service=service, name="SMS") + + url, data = component_form_data(staff_client, child, parent=str(parent.pk)) + assert staff_client.post(url, data).status_code == 302 + + child.refresh_from_db() + assert child.ancestor_ids == [parent.pk] + assert list(ServiceComponent.objects.search("programmable sms")) == [child] From c3797c14ca8697466c356bf0fc4ea11a77bb3ef7 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 21:36:50 -0700 Subject: [PATCH 22/94] fix: a ?fields= path that names no field answers 400 `_prune` popped every field a name in the tree did not match, and checked no name against the serializer. `?fields=total_nonsense` then returned `200 {}`. The contract has always promised 400 here. `fields` is on all twelve GET operations and is the parameter a generated client uses most. A typo, or a field a later change renames, returned an empty payload with nothing to say why. Nested branches inherit the check through the recursive prune. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/common/mixins.py | 14 ++++++++++++++ api/tests/test_common_api.py | 9 ++++++--- api/tests/test_contract.py | 12 ++++++++++++ docs/api/openapi.yaml | 3 ++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/api/common/mixins.py b/api/common/mixins.py index ea9b8af..a4461ee 100644 --- a/api/common/mixins.py +++ b/api/common/mixins.py @@ -1,3 +1,5 @@ +from rest_framework.serializers import ValidationError + _UNSET = object() @@ -52,6 +54,18 @@ def _parse(raw): return tree def _prune(self, tree): + unknown = sorted(set(tree) - set(self.fields)) + if unknown: + # A name nothing serves used to pop every field and answer + # `200 {}`. A typo, or a field a later change renames, then + # read as a server with nothing to say. + raise ValidationError( + { + self.fields_param or "fields": [ + f"Unknown field: {name}." for name in unknown + ] + } + ) for name in set(self.fields) - set(tree): self.fields.pop(name) for name, children in tree.items(): diff --git a/api/tests/test_common_api.py b/api/tests/test_common_api.py index 32ee099..80717f4 100644 --- a/api/tests/test_common_api.py +++ b/api/tests/test_common_api.py @@ -39,9 +39,12 @@ def test_a_dotted_path_prunes_inside_a_nested_object_rather_than_dropping_it(): assert set(result["status"]) == {"severity"} -def test_an_unknown_field_is_ignored_rather_than_erroring(): - # A client asking for a field that does not exist yet gets what does exist. - assert set(_serialize("id,nonsense")) == {"id"} +def test_an_unknown_field_is_refused(): + # Ignoring it popped every field and returned an empty object. A + # client then read a typo as a server holding nothing. + with pytest.raises(serializers.ValidationError) as raised: + _serialize("id,nonsense") + assert raised.value.detail == {"fields": ["Unknown field: nonsense."]} def test_a_serializer_can_opt_out(): diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py index 9a94e0e..7aac92d 100644 --- a/api/tests/test_contract.py +++ b/api/tests/test_contract.py @@ -190,6 +190,18 @@ def test_fields_prunes_the_response(path): assert set(kept) == {field}, f"GET {path} ignored ?fields=: {sorted(kept)}" +@pytest.mark.django_db +@pytest.mark.parametrize("path", sorted(FIELDS_PROBES)) +def test_fields_refuses_a_name_that_is_no_field(path): + # Nothing checked that a name in the tree named a field. An unknown + # one popped them all and answered `200 {}`. The contract promises + # 400. A typo must not read as a server with nothing to say. + client, url, _field = FIELDS_PROBES[path]() + response = client.get(url, {"fields": "total_nonsense"}) + assert response.status_code == 400, f"GET {path} answered {response.status_code}" + assert response.json() == {"fields": ["Unknown field: total_nonsense."]} + + @pytest.mark.django_db def test_the_component_schema_has_exactly_the_documented_fields(committed): from catalog.serializers import ComponentSerializer diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index d40fc0d..0cd3450 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -452,7 +452,8 @@ components: Use a dotted path for a nested field. `overall_component.status.severity` returns `overall_component` containing `status` containing `severity` only. - A path that names an unknown field returns `400`. + A path that names an unknown field returns `400`. The body is a field-errors object, + one array of messages under `fields` — not the `Error` shape. PageSize: { name: page_size, in: query, schema: { type: integer, default: 50, minimum: 1, maximum: 200 } } Slug: { name: slug, in: path, required: true, schema: { type: string }, example: twilio } Uuid: { name: uuid, in: path, required: true, schema: { type: string, format: uuid } } From 2f9f74aef6a9284613c32360eb82bc9f7d0fbc1c Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 21:40:00 -0700 Subject: [PATCH 23/94] docs: the contract describes the API the branch left behind The "declared, not generated" block still named `overall_component__is_tracked`, which belonged to the service list this branch removed. It counted three declared parameters where there are eleven. It told a client to read a field's path in the response body to build a filter, which is now false for the flagship endpoints. `service` takes a slug where the body nests a `ServiceRef`. `component` and `ancestor` name nothing in any body. `phase` takes `open` where the body carries `investigating`. Four smaller disagreements go with it. The `catalog` tag still claimed incidents, which moved to `status`. `StatusAggregates` still described a service list. `upcoming_maintenance_count` pointed at a call returning every window ever, missing `phase=open`. `POST /catalog/requests/` denied the optional auth its own test asserts. The `fields` parameter now says a dotted path reaches into `enums` on `/meta/`. That behaviour exists because a user reported the parameter doing nothing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- docs/api/openapi.yaml | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 0cd3450..d1b2fb4 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -69,17 +69,25 @@ info: `allOf` appears where two schemas cover two models. It also appears where a detail schema layers that kind of value onto its own list row. - Query parameters use the ORM path. django-filter generates them from `filterset_fields` and - `ordering_fields`. No `FilterSet` class is necessary. + Query parameters are declared in a `FilterSet`. Almost none is generated from a column of + the same name. `is_overall` and `kind` are the two that are. - The nesting is visible in the URL. Read a field's path in the response body to build its - filter. This holds for fields added later. + So a parameter is not an ORM path, and a field's path in the response body does not build + its filter. `service` takes a slug where the body carries a nested `ServiceRef`. `component` + and `ancestor` name nothing in any body. `phase` takes `open` or `closed` where the body + carries `investigating`. - Three parameters are declared, not generated. They have no ORM path: - `overall_component__is_tracked`, `ordering=suggested`, and `ordering=next_transition`. + Read the operation's own parameter list instead. It says what each one takes and what it + narrows. - Model field names are chosen to read well in a URL, because a field name appears in `?fields=` - and in a filter path. + Eleven are declared across the two collections: `q`, `service`, `ancestor`, `event`, + `component`, `dashboard`, `is_tracked`, `phase`, and the three `status__severity` forms. + + Two ordering values are declared as well, `suggested` and `next_transition`. Neither names a + column. A view annotates the fields each one sorts on. + + Model field names are chosen to read well in a URL, because a field name appears in + `?fields=`. ## Conventions @@ -103,7 +111,7 @@ tags: - name: auth - name: me - name: catalog - description: Public. Services, their components, and incidents. + description: Public. Services and their components. - name: status description: Public. The event feed, one collection for Home, a service and a component. - name: board @@ -223,7 +231,9 @@ paths: holds the URL, so asking cannot reveal what we have. One row per URL, counted. Throttled per caller. - security: [] + + Signing in is optional. A signed-in asker is recorded on the row. + security: [{}, { bearer: [] }] requestBody: required: true content: @@ -452,6 +462,9 @@ components: Use a dotted path for a nested field. `overall_component.status.severity` returns `overall_component` containing `status` containing `severity` only. + A dotted path reaches into `enums` on `/meta/` too. `enums.severity` returns that one + label map. `enums` alone returns all of them. + A path that names an unknown field returns `400`. The body is a field-errors object, one array of messages under `fields` — not the `Error` shape. PageSize: { name: page_size, in: query, schema: { type: integer, default: 50, minimum: 1, maximum: 200 } } @@ -574,7 +587,7 @@ components: allOf: - $ref: '#/components/schemas/Aggregates' - type: object - description: For any list of things that carry a status — services, components, a board. + description: For any list of components. Discover, a service's parts, a board. properties: by_severity: type: object @@ -840,7 +853,8 @@ components: The full list is not on the component. History is unbounded, and a row renders from the soonest plus this count. Read the list from - `GET /events/?component={uuid}&kind=maintenance` when a screen needs it. + `GET /events/?component={uuid}&kind=maintenance&phase=open` when a screen needs it. + Without `phase=open` that call returns every window ever, which this never counts. active_incident: allOf: [{ $ref: '#/components/schemas/EventRef' }] description: | From 9b9d7c2422fad94d4e7e1e2cd1f70dd4a8bcd177 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 21:41:55 -0700 Subject: [PATCH 24/94] ci: the merge gate runs the bin/ tests `just check` runs them. The workflow enumerates its own steps and had no equivalent, so the tests moved out of the API suite ran only on a laptop. CI is what a merge waits on, so they gated nothing. They are stdlib only, so they run on the runner beside the prose check rather than paying for a container. A new bin test holds the two lists together. It reads the `check` recipe and the workflow, and fails when one gains a check the other does not have. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- .github/workflows/ci.yml | 5 ++ bin/tests/test_merge_gate.py | 94 ++++++++++++++++++++++++++++++++++++ justfile | 5 +- 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 bin/tests/test_merge_gate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dc22fd..2ff88ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,11 @@ jobs: - name: Prose run: python3 bin/check_prose.py + # Stdlib only as well. These cover the scripts in bin/, which have + # no dependency manager and no Django to test them with. + - name: The bin/ scripts + run: python3 -m unittest discover -s bin/tests + - name: No migration is missing run: | docker compose -f docker-compose.test.yml run --rm --entrypoint sh pytest \ diff --git a/bin/tests/test_merge_gate.py b/bin/tests/test_merge_gate.py new file mode 100644 index 0000000..7e949f1 --- /dev/null +++ b/bin/tests/test_merge_gate.py @@ -0,0 +1,94 @@ +"""`just check` and the CI workflow must gate on the same checks. + +`bin/` has no dependency manager and no Django. These tests use only the +standard library, so they run on a machine that has done nothing but +`just init`. Run them with `python3 -m unittest discover -s bin/tests`. +""" + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +JUSTFILE = ROOT / "justfile" +WORKFLOW = ROOT / ".github" / "workflows" / "ci.yml" + + +def recipe(name): + """One recipe's commands, with a line continuation joined up.""" + lines = [] + inside = False + for line in JUSTFILE.read_text().splitlines(): + if re.match(rf"^{re.escape(name)}(\s|:)", line): + inside = True + continue + if not inside: + continue + if line and not line[0].isspace(): + break + body = line.strip() + if not body or body.startswith("#"): + continue + if lines and lines[-1].endswith("\\"): + lines[-1] = lines[-1][:-1] + body + else: + lines.append(body) + return lines + + +def commands(): + """Every command `just check` runs, with a called recipe inlined.""" + for line in recipe("check"): + called = re.match(r"@?just\s+([\w-]+)", line) + yield from recipe(called.group(1)) if called else [line] + + +def compose_services(command): + """The service a `docker compose run` names, past its flags.""" + _, _, tail = command.partition("run --rm ") + tokens = tail.split() + while tokens: + token = tokens.pop(0) + if token == "--entrypoint": + tokens.pop(0) + elif not token.startswith("-"): + return [token] + return [] + + +class MergeGateTests(unittest.TestCase): + """CI is the gate a merge waits on. `just check` is not. + + The bin/ tests ran in `just check` and in nothing CI did. So they + gated nothing, which is how they came to run only on a laptop. + """ + + def setUp(self): + self.workflow = WORKFLOW.read_text() + # Teardown, not a check. It proves nothing about the code. + self.commands = [c for c in commands() if "down -v" not in c] + + def test_every_container_check_also_runs_in_ci(self): + for command in self.commands: + for service in compose_services(command): + self.assertIn( + f"run --rm {service}", self.workflow, f"CI skips {service}" + ) + + def test_every_host_check_also_runs_in_ci(self): + for command in self.commands: + if "docker compose" in command: + continue + for script in re.findall(r"bin/[\w./-]+", command): + self.assertIn(script, self.workflow, f"CI skips {script}") + + def test_the_migration_check_also_runs_in_ci(self): + # It runs through an entrypoint override, so the service name + # alone does not say the check itself is there. + wanted = [c for c in self.commands if "makemigrations" in c] + self.assertTrue(wanted, "just check no longer checks for a missing migration") + self.assertIn("makemigrations --check --dry-run", self.workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/justfile b/justfile index 167eae8..ee881ec 100644 --- a/justfile +++ b/justfile @@ -101,8 +101,11 @@ test-bin: # Slower than `just test`, and the answer that counts: it proves the # image rather than the host. Run it before opening a pull request. +# The two stdlib-only checks run on the host, because a container costs +# more than it saves. `bin/tests/test_merge_gate.py` holds the two lists +# together. -# Everything CI runs, in the containers CI runs it in. +# Every check CI runs, each where CI runs it. check: image docker compose -f docker-compose.test.yml run --rm pytest -q -n auto --cov-fail-under=85 docker compose -f docker-compose.test.yml run --rm ruff From 0022eca9351971ffece1296c74476ebf2a373048 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 21:45:16 -0700 Subject: [PATCH 25/94] test: a ranked search pages without repeating a row `?q=` is Discover's default path and the most used list in the app. `rank` is a float annotation with heavy ties, because identical documents rank identically. A cursor over a non-unique key repeats or skips a row, and infinite scroll shows it to a reader who never files it. Nothing covered that path. Two settings go with it. `SEARCH_PARAM` named the parameter for DRF's `SearchFilter`, which no view uses any more: `ComponentListView` reads `q` by hand. The setting was a trap, because changing it moved nothing. `track()` minted a watcher address from a counter that resets per test, where `UserFactory` holds one that does not. Email is unique, so the first test to call both would raise far from the cause. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/settings.py | 3 --- api/tests/factories.py | 11 ++++++----- api/tests/test_component_api.py | 25 +++++++++++++++++++++++++ api/tests/test_dashboards_models.py | 14 ++++++++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/api/api/settings.py b/api/api/settings.py index 0b40725..4b21865 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -147,9 +147,6 @@ "rest_framework.filters.OrderingFilter", "common.filters.FieldsBackend", ], - # The contract calls it `q`, and DRF's SearchFilter calls it - # `search`. One name, and the schema documents it. - "SEARCH_PARAM": "q", # The contract has always answered 429, and nothing raised one. See # `Throttle` for what each rate is protecting. "DEFAULT_THROTTLE_CLASSES": [ diff --git a/api/tests/factories.py b/api/tests/factories.py index 8356261..13d494c 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -74,15 +74,16 @@ class Meta: def track(component, user=None): - """Track a component, the way somebody using the app would.""" - from django.contrib.auth import get_user_model + """Track a component, the way somebody using the app would. + The user comes from `UserFactory`, which owns the one counter behind + these addresses. A second counter here reset per test and collided + with the factory's, on a column the database holds unique. + """ from dashboards.models import Dashboard, DashboardItem if user is None: - user = get_user_model().objects.create( - email=f"watcher{DashboardItem.objects.count()}@example.test" - ) + user = UserFactory() board = user.dashboards.first() or Dashboard.objects.create(owner=user) return DashboardItem.objects.create(dashboard=board, component=component) diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py index 3e7a21a..f8747c2 100644 --- a/api/tests/test_component_api.py +++ b/api/tests/test_component_api.py @@ -167,3 +167,28 @@ def test_event_narrows_to_the_components_one_event_affects(client, tree): event.affected_components.add(parent) response = client.get(reverse("component-list"), {"event": str(event.id)}) assert [r["name"] for r in response.json()["results"]] == ["Programmable Messaging"] + + +def test_a_ranked_search_pages_without_repeating_a_row(client): + """`?q=` is Discover's default path, and `rank` ties heavily. + + Identical documents rank identically. `EnvelopePagination` appends + `-created_at` to make the key unique. Without it a cursor repeats or + skips a row, and infinite scroll shows it. + """ + service = ServiceFactory(name="Twilio") + for _ in range(7): + ComponentFactory(service=service, name="SMS") + rebuild_ancestry(service) + rebuild_search(service) + + seen = [] + url = reverse("component-list") + params = {"q": "twilio sms", "page_size": 2} + while url: + body = client.get(url, params).json() + seen += [row["id"] for row in body["results"]] + url, params = body["next"], {} + + assert seen == list(dict.fromkeys(seen)), "a row came back on two pages" + assert len(seen) == 7, "a row was never reached" diff --git a/api/tests/test_dashboards_models.py b/api/tests/test_dashboards_models.py index ef2ddf1..4b57050 100644 --- a/api/tests/test_dashboards_models.py +++ b/api/tests/test_dashboards_models.py @@ -152,3 +152,17 @@ def test_closing_an_account_still_takes_its_boards(): user.delete() assert not Dashboard.objects.filter(owner_id=user.pk).exists() + + +@pytest.mark.django_db +def test_a_factory_user_and_a_tracked_one_never_collide(): + # `track` minted its own address from a counter that resets per + # test. UserFactory's persists across the session, and email is + # unique. The first test to call both would raise far from the cause. + from tests.factories import UserFactory, track + + UserFactory.reset_sequence(0) + UserFactory() + track(ComponentFactory()) + + assert User.objects.filter(email__startswith="watcher").count() == 2 From ae9e90fd2205facd12dbe9bfd7fc9ed3ed9cf010 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 22:13:29 -0700 Subject: [PATCH 26/94] fix: an archived component is not served anywhere A provider drops a component and the row is archived, not deleted. Three readers then disagreed about it. `Service.component_count` left it out. `descendant_count`, the component collection, the component detail and the board all returned it. So a service page showed a badge counting one set of components and a list under it showing another. The spec says of those two screens that they cannot count differently. Archived components are now left out of every list, every count and the detail, which answers 404. The rule is `ServiceComponentQuerySet.visible` and every reader calls it. Restating the filter per reader is how two answers to one question drift apart. It is not part of `for_display`. That queryset also prefetches a service's rollup, and a service whose rollup is archived still needs a status in its header. That stays the one place an archived component reaches a caller. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/models.py | 30 +++++++++++++++------ api/catalog/views_components.py | 3 ++- api/dashboards/views.py | 2 +- api/status/serializers.py | 4 ++- api/tests/test_board_api.py | 15 +++++++++++ api/tests/test_catalog_models.py | 45 ++++++++++++++++++++++++++++++++ api/tests/test_component_api.py | 24 +++++++++++++++++ api/tests/test_event_api.py | 17 ++++++++++++ docs/api/openapi.yaml | 43 +++++++++++++++++++++++++----- 9 files changed, 165 insertions(+), 18 deletions(-) diff --git a/api/catalog/models.py b/api/catalog/models.py index f279c0d..60cc424 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -34,9 +34,7 @@ def for_display(self, user=None): self.select_related("status_page", "poller") .annotate( _component_count=related_count( - ServiceComponent.objects.filter( - is_overall=False, is_archived=False - ), + ServiceComponent.objects.visible().filter(is_overall=False), "service", ), _tracked_component_count=tracked, @@ -93,7 +91,7 @@ def component_count(self): prepared = getattr(self, "_component_count", None) if prepared is not None: return prepared - return self.components.filter(is_overall=False, is_archived=False).count() + return self.components.visible().filter(is_overall=False).count() def tracked_component_count(self, user): """How many parts this user has on a board.""" @@ -247,7 +245,8 @@ def _descendant_count(): output_field=ArrayField(models.UUIDField()), ) counted = ( - ServiceComponent.objects.filter(ancestor_ids__contains=key) + ServiceComponent.objects.visible() + .filter(ancestor_ids__contains=key) .order_by() .values(one_group=models.Value(1)) .annotate(total=models.Count("pk")) @@ -257,6 +256,19 @@ def _descendant_count(): class ServiceComponentQuerySet(models.QuerySet): + def visible(self): + """The components a caller is served, anywhere. + + The provider stopped publishing an archived one. Every list, + every count and the detail read this, so no two of them can + answer the same question differently. + + It is not part of `for_display`. That queryset also prefetches a + service's rollup, and a service whose rollup is archived still + needs a header status. + """ + return self.filter(is_archived=False) + def search(self, q): """Rank a query against the stored path. @@ -411,9 +423,11 @@ def descendant_count(self): return 0 prepared = getattr(self, "_descendant_count", None) if prepared is None: - return ServiceComponent.objects.filter( - ancestor_ids__contains=[self.pk] - ).count() + return ( + ServiceComponent.objects.visible() + .filter(ancestor_ids__contains=[self.pk]) + .count() + ) return prepared def is_tracked_by(self, user): diff --git a/api/catalog/views_components.py b/api/catalog/views_components.py index ec96e9b..e005c56 100644 --- a/api/catalog/views_components.py +++ b/api/catalog/views_components.py @@ -28,7 +28,8 @@ class ComponentQueryMixin: def get_queryset(self): return ( - ServiceComponent.objects.for_display(self.request.user) + ServiceComponent.objects.visible() + .for_display(self.request.user) .annotate( severity_now=CURRENT_SEVERITY, watcher_count=COMPONENT_WATCHER_COUNT, diff --git a/api/dashboards/views.py b/api/dashboards/views.py index c24f3d3..4777b6b 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -40,7 +40,7 @@ class BoardComponentListView(generics.ListCreateAPIView): ordering = ["severity_now"] # worst first: lower severity is worse def get_queryset(self): - rows = ServiceComponent.objects.annotate( + rows = ServiceComponent.objects.visible().annotate( severity_now=CURRENT_SEVERITY, next_transition=NEXT_TRANSITION ) if "uuid" not in self.kwargs: diff --git a/api/status/serializers.py b/api/status/serializers.py index eb25847..80a4ca4 100644 --- a/api/status/serializers.py +++ b/api/status/serializers.py @@ -92,7 +92,9 @@ def get_update_count(self, event): @extend_schema_field(serializers.IntegerField()) def get_affected_count(self, event): - return event.affected_components.count() + # `visible`, because the Affects tab reads `?event=` and counts + # the same rows. The badge cannot say 2 above a list of one. + return event.affected_components.visible().count() @extend_schema_field(serializers.DateTimeField(allow_null=True)) def get_last_update_at(self, event): diff --git a/api/tests/test_board_api.py b/api/tests/test_board_api.py index 3264d7c..b215fd6 100644 --- a/api/tests/test_board_api.py +++ b/api/tests/test_board_api.py @@ -70,6 +70,21 @@ def test_all_returns_every_tracked_component(client, board): assert client.get(url).json()["aggregates"]["total"] == 2 +@pytest.mark.django_db +def test_the_board_leaves_out_an_archived_component(client, board): + # The provider stopped publishing it, so no screen shows it. A board + # row that no other list holds would have nowhere to link to. + _track(board) + gone = _track(board) + gone.is_archived = True + gone.save(update_fields=["is_archived"]) + + url = reverse("board-components", kwargs={"uuid": board.id}) + body = client.get(url).json() + assert body["aggregates"]["total"] == 1 + assert str(gone.id) not in [r["id"] for r in body["results"]] + + @pytest.mark.django_db def test_one_fetch_fills_every_chip(client, board): # One response must fill every chip on the board. diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 65b6673..c59ddc5 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -194,6 +194,51 @@ def test_for_display_counts_descendants_without_a_query_a_row( assert counts == [0, 1, 2] +def test_the_prepared_descendant_count_leaves_out_an_archived_row(db): + # The badge and the list below it read one number. If this counted + # an archived row, the tab would say 2 and then show one component. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + # `update_fields`, because this row is stale: `rebuild_ancestry` + # wrote its ancestry after the factory made it. + leaf.is_archived = True + leaf.save(update_fields=["is_archived"]) + + row = ServiceComponent.objects.for_display().get(pk=top.pk) + assert row.descendant_count() == 1 + + +def test_the_unprepared_descendant_count_leaves_out_an_archived_row(db): + # A row fetched without `for_display` counts for itself. Two ways to + # answer one question is how the two answers drift apart. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + leaf.is_archived = True + leaf.save(update_fields=["is_archived"]) + + assert ServiceComponent.objects.get(pk=top.pk).descendant_count() == 1 + assert middle.descendant_count() == 0 + + +def test_a_services_component_count_leaves_out_an_archived_row(db): + # This is the badge on the service page. The Components tab under it + # lists the same rows, and the two cannot count differently. + service = ServiceFactory() + ComponentFactory(service=service) + gone = ComponentFactory(service=service) + gone.is_archived = True + gone.save() + + assert Service.objects.for_display().get(pk=service.pk).component_count() == 1 + assert Service.objects.get(pk=service.pk).component_count() == 1 + + def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): # The rollup's own name is an exact, weight-A hit. A leaf matches # only through its service, at weight C, so it must sort below. diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py index f8747c2..950c1a1 100644 --- a/api/tests/test_component_api.py +++ b/api/tests/test_component_api.py @@ -102,6 +102,30 @@ def test_the_detail_answers_by_uuid(client, tree): assert response.json()["name"] == "SMS" +def test_the_collection_leaves_out_an_archived_component(client, tree): + # A provider dropped it, so nothing lists it. The service badge + # counts the same rows, and the two cannot count differently. + _, _, _, leaf = tree + # `update_fields`, because the fixture rebuilt this row's ancestry + # and its search document after the factory made it. + leaf.is_archived = True + leaf.save(update_fields=["is_archived"]) + + body = client.get(reverse("component-list")).json() + assert body["aggregates"]["total"] == 2 + assert "SMS" not in [r["name"] for r in body["results"]] + + +def test_the_detail_refuses_an_archived_component(client, tree): + # Archived components are not served anywhere. A row the list hides + # and the detail answers would be two rules, not one. + _, _, _, leaf = tree + leaf.is_archived = True + leaf.save(update_fields=["is_archived"]) + + assert client.get(reverse("component-detail", args=[leaf.id])).status_code == 404 + + def test_a_query_keeps_the_rank_the_search_worked_out(client, tree): # The rollup carries the word in its own name. The other two match # only through their service. A default sort applied after the diff --git a/api/tests/test_event_api.py b/api/tests/test_event_api.py index f943a12..6885b00 100644 --- a/api/tests/test_event_api.py +++ b/api/tests/test_event_api.py @@ -239,6 +239,23 @@ def test_the_detail_carries_both_tab_counts(client): assert body["affected_count"] == 2 +def test_the_affects_badge_leaves_out_an_archived_component(client): + # The badge and the Affects tab count the same rows. The tab reads + # `?event=`, which does not serve an archived component. + service = ServiceFactory() + kept = ComponentFactory(service=service) + gone = ComponentFactory(service=service) + event = _event(service, kept, external_id="1") + event.affected_components.add(gone) + gone.is_archived = True + gone.save() + + badge = client.get(reverse("event-detail", args=[event.id])).json() + listed = client.get(reverse("component-list"), {"event": str(event.id)}).json() + assert badge["affected_count"] == 1 + assert listed["aggregates"]["total"] == 1 + + def test_the_timeline_is_its_own_paged_list(client): # A provider's log has no ceiling, so it cannot ride on the detail. service = ServiceFactory() diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index d1b2fb4..dbdbff2 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -13,6 +13,21 @@ info: To track a service, track its overall component. To track one part of a service, track that component. Both are the same call with a different id. + ## An archived component is not served + + A provider stops publishing a component. The row is archived rather than deleted, because + somebody may be tracking it. + + Archived components are left out of every list, every count and the detail. The detail + answers `404`. Nothing points at a row no list holds. + + The one exception is `Service.overall_component`. A service whose rollup is archived still + needs a status in its header. + + Two consequences follow. A tracked component that is archived leaves the board with no + notice. An event names the components it affected, and a link to an archived one answers + `404`. + ## Severity is the state `severity` is an integer from 0 to 5. Lower is worse. This matches the SEV0/SEV1 convention. @@ -264,6 +279,9 @@ paths: `q` matches a component's whole path, so `twilio` finds `SMS`. Results are then ranked by relevance. Without `q` the default is `suggested`, except under `service`, where it is `status_page_order`. + + Archived components are never listed. An event whose only affected component was archived + returns an empty page. security: [{}, { bearer: [] }] parameters: - { name: q, in: query, schema: { type: string } } @@ -296,6 +314,9 @@ paths: get: tags: [catalog] summary: Component detail + description: | + `404` for an archived component, as well as for one that never existed. A provider + stopped publishing it, so no screen holds it. security: [{}, { bearer: [] }] parameters: - { $ref: '#/components/parameters/Uuid' } @@ -389,7 +410,12 @@ paths: get: tags: [board] summary: The components tracked on this dashboard - description: Filtering, sorting and pagination are server-side without exception — nothing assumes a board is small. + description: | + Filtering, sorting and pagination are server-side without exception. Nothing assumes a + board is small. + + An archived component drops off the board. The item stays tracked, so the row returns if + the provider publishes the component again. parameters: - { $ref: '#/components/parameters/Uuid' } - { name: status__severity__lte, in: query, schema: { type: integer } } @@ -763,10 +789,13 @@ components: rather than a copy of its fields. One FK, one nested serializer. It is a plain `Component`, the same shape a component has anywhere else, so one - renderer draws every row. Six of its fields are fixed by what an overall component is: - `parent` and `path` are null, `descendant_count` is 0, `is_overall` is true, `archived_at` - is null, and `name` is the service name. They are returned rather than omitted — a - client that special-cased them would need a second renderer to gain nothing. + renderer draws every row. Five of its fields are fixed by what an overall component is: + `parent` and `path` are null, `descendant_count` is 0, `is_overall` is true, and `name` + is the service name. They are returned rather than omitted. A client that + special-cased them would need a second renderer to gain nothing. + + This is the one place an archived component is served. A provider can stop publishing + its top-level indicator, and the header still needs a status. Read `archived_at` here. Read severity at `overall_component.status.severity`. The filter uses the same path. tracked_component_count: @@ -821,7 +850,7 @@ components: Always 0 on the overall component. It is a peer of the top-level components, not their parent. is_overall: { type: boolean, description: The component Statusboard creates per service, carrying the provider's top-level indicator. Always first, parent is null. } - archived_at: { type: string, format: date-time, nullable: true, description: Set when the provider stops publishing it. The row stays, reads unknown, and says so — deleting it would silently drop it from someone's board. } + archived_at: { type: string, format: date-time, nullable: true, description: 'When the provider stopped publishing the component. An archived component is not served, so this is null everywhere but `Service.overall_component`.' } status: { $ref: '#/components/schemas/Status' } upcoming_maintenance: allOf: [{ $ref: '#/components/schemas/EventRef' }] @@ -937,7 +966,7 @@ components: badges before either tab has made a request. properties: update_count: { type: integer } - affected_count: { type: integer } + affected_count: { type: integer, description: 'The Affects badge. Archived components are left out, the same as the tab it labels. It can be 0.' } last_update_at: { type: string, format: date-time, nullable: true } EventUpdate: type: object From c5f56023adb60ff87711057cef62cfd67c110411 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 22:37:16 -0700 Subject: [PATCH 27/94] fix: archived components leave every count and refuse a track `Service.tracked_component_count` counted archived rows, in the annotation and in the fallback. A user whose only tracked component was archived saw a count of one, read that as "this service is on my board", and then got no such row from the board list. `POST /dashboards/{uuid}/components/` looked the component up unfiltered. `Service.overall_component` is the one place an archived component still reaches a client, so the call was reachable without a stale id. It made a board row that no list renders and nothing can offer to untrack. It answers 404 now, the same as the component detail answers for that id. The contract also claimed nothing points at a row no list holds. `Component.path` and `Component.parent` do. The sentence was the defect, not the behaviour, so it now says what a client should draw instead. The board POST documented a 409 the code has never returned. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/models.py | 10 ++++++---- api/dashboards/views.py | 6 +++++- api/tests/test_board_api.py | 12 ++++++++++++ api/tests/test_catalog_models.py | 16 ++++++++++++++++ docs/api/openapi.yaml | 10 ++++++---- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/api/catalog/models.py b/api/catalog/models.py index 60cc424..5e7d0fc 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -28,7 +28,8 @@ def for_display(self, user=None): tracked = models.Value(0, output_field=models.IntegerField()) if user is not None and not isinstance(user, AnonymousUser): tracked = related_count( - ServiceComponent.objects.filter(boards__owner=user), "service" + ServiceComponent.objects.visible().filter(boards__owner=user), + "service", ) return ( self.select_related("status_page", "poller") @@ -101,7 +102,8 @@ def tracked_component_count(self, user): if prepared is not None: return prepared return ( - ServiceComponent.objects.filter(service=self, boards__owner=user) + ServiceComponent.objects.visible() + .filter(service=self, boards__owner=user) .distinct() .count() ) @@ -260,8 +262,8 @@ def visible(self): """The components a caller is served, anywhere. The provider stopped publishing an archived one. Every list, - every count and the detail read this, so no two of them can - answer the same question differently. + every count, the detail and the track write read this. So no two + of them can answer the same question differently. It is not part of `for_display`. That queryset also prefetches a service's rollup, and a service whose rollup is archived still diff --git a/api/dashboards/views.py b/api/dashboards/views.py index 4777b6b..f2f32db 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -62,12 +62,16 @@ def get_queryset(self): response=ComponentSerializer, description="Already tracked. Tracking twice is not an error.", ), + 404: OpenApiResponse(description="No such board, or no such component."), }, ) def create(self, request, *args, **kwargs): board = _board(request, self.kwargs["uuid"]) + # `visible`, so a write agrees with the reads. An archived id + # already answers 404 on the component detail. Accepting it here + # made a board row that no list renders. component = get_object_or_404( - ServiceComponent, id=request.data.get("component_id") + ServiceComponent.objects.visible(), id=request.data.get("component_id") ) _, created = DashboardItem.objects.get_or_create( dashboard=board, diff --git a/api/tests/test_board_api.py b/api/tests/test_board_api.py index b215fd6..756a610 100644 --- a/api/tests/test_board_api.py +++ b/api/tests/test_board_api.py @@ -163,6 +163,18 @@ def test_tracking_the_same_component_twice_is_not_an_error(client, board): assert DashboardItem.objects.filter(dashboard=board).count() == 1 +@pytest.mark.django_db +def test_tracking_an_archived_component_is_refused(client, board): + # `Service.overall_component` still hands a client an archived + # rollup, so this needs no stale id. Accepting it made a board row + # that no list renders and nothing can offer to untrack. + component = ComponentFactory(service=ServiceFactory(), is_archived=True) + url = reverse("board-components", kwargs={"uuid": board.id}) + response = client.post(url, {"component_id": str(component.id)}, format="json") + assert response.status_code == 404 + assert not DashboardItem.objects.filter(dashboard=board).exists() + + @pytest.mark.django_db def test_stopping_tracking_removes_the_item(client, board): component = _track(board) diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index c59ddc5..fe8a369 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -239,6 +239,22 @@ def test_a_services_component_count_leaves_out_an_archived_row(db): assert Service.objects.get(pk=service.pk).component_count() == 1 +def test_a_services_tracked_count_leaves_out_an_archived_row(db): + # The client reads a count above zero as "this service is on your + # board". The board list leaves an archived row out. A count that + # held one promised a row the board never returns. + user = UserFactory() + service = ServiceFactory() + gone = ComponentFactory(service=service) + track(gone, user=user) + gone.is_archived = True + gone.save() + + prepared = Service.objects.for_display(user).get(pk=service.pk) + assert prepared.tracked_component_count(user) == 0 + assert Service.objects.get(pk=service.pk).tracked_component_count(user) == 0 + + def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): # The rollup's own name is an exact, weight-A hit. A leaf matches # only through its service, at weight C, so it must sort below. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index dbdbff2..cc562d2 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -19,14 +19,15 @@ info: somebody may be tracking it. Archived components are left out of every list, every count and the detail. The detail - answers `404`. Nothing points at a row no list holds. + answers `404`, and so does tracking one. The one exception is `Service.overall_component`. A service whose rollup is archived still needs a status in its header. - Two consequences follow. A tracked component that is archived leaves the board with no + Three consequences follow. A tracked component that is archived leaves the board with no notice. An event names the components it affected, and a link to an archived one answers - `404`. + `404`. `Component.path` and `Component.parent` name ancestors without checking them, so a + breadcrumb step can answer `404` too. Render a node you cannot fetch as plain text. ## Severity is the state @@ -457,7 +458,8 @@ paths: schema: { type: object, required: [component_id], properties: { component_id: { type: string, format: uuid } } } responses: '201': { description: Created, content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } - '409': { description: Already tracked } + '200': { description: 'Already tracked. Tracking twice is not an error.', content: { application/json: { schema: { $ref: '#/components/schemas/Component' } } } } + '404': { description: 'No such board, or no such component. An archived component is not tracked.' } /dashboards/{uuid}/components/{component_id}/: delete: From 4b6abfd0457a408d5076afd134ed816ef5433ef6 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 22:39:05 -0700 Subject: [PATCH 28/94] fix: a dotted path is refused wherever it names nothing The contract promises `400` for a path naming an unknown field, because a silently empty payload is the failure a user reported on `/meta/`. `FieldsMixin._prune` raises, and two readers went around it. `enums` is a plain dict, so `/meta/` filters it by hand and had no unknown-name check. `?fields=enums.nonsense` answered `200 {"enums": {}}`, the exact endpoint and failure class the rule exists for. `Component.path` is a method field, so the recursion could not reach it. A dotted path into a step was neither pruned nor refused. The step serializer now prunes like every other nested one. The refusal itself moves into `FieldsMixin.reject_unknown`, so a reader that prunes by hand asks the same rule rather than restating it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/serializers.py | 9 +++++++-- api/common/mixins.py | 21 ++++++++++++++++----- api/common/serializers.py | 5 +++++ api/tests/test_catalog_api.py | 30 ++++++++++++++++++++++++++++++ api/tests/test_meta.py | 10 ++++++++++ 5 files changed, 68 insertions(+), 7 deletions(-) diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index dc5c18e..30a0289 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -38,7 +38,7 @@ class Meta: fields = ["id", "slug", "name", "logo"] -class PathNodeSerializer(serializers.ModelSerializer): +class PathNodeSerializer(FieldsMixin, serializers.ModelSerializer): """One step above a component, enough to link to it. A joined string named the ancestors and gave a client nothing to @@ -105,7 +105,12 @@ def get_path(self, row): # did not. if row.is_overall: return [] - return PathNodeSerializer(row.ancestors, many=True).data + return PathNodeSerializer( + row.ancestors, + many=True, + context=self.context, + fields_tree=self.child_tree("path"), + ).data @extend_schema_field(serializers.IntegerField()) def get_descendant_count(self, row): diff --git a/api/common/mixins.py b/api/common/mixins.py index a4461ee..edac56a 100644 --- a/api/common/mixins.py +++ b/api/common/mixins.py @@ -53,12 +53,20 @@ def _parse(raw): branch.add(tail) return tree - def _prune(self, tree): - unknown = sorted(set(tree) - set(self.fields)) + def reject_unknown(self, names, known): + """Refuse a path that names nothing, for a reader that prunes itself. + + A name nothing serves used to pop every field and answer + `200 {}`. A typo, or a field a later change renames, then read + as a server with nothing to say. + + `_prune` recurses into a nested serializer. A plain dict and a + method field hold no `fields`, so the recursion stops before + them. Such a reader filters by hand and calls this. One rule + then answers for every place a dotted path is accepted. + """ + unknown = sorted(set(names) - set(known)) if unknown: - # A name nothing serves used to pop every field and answer - # `200 {}`. A typo, or a field a later change renames, then - # read as a server with nothing to say. raise ValidationError( { self.fields_param or "fields": [ @@ -66,6 +74,9 @@ def _prune(self, tree): ] } ) + + def _prune(self, tree): + self.reject_unknown(tree, self.fields) for name in set(self.fields) - set(tree): self.fields.pop(name) for name, children in tree.items(): diff --git a/api/common/serializers.py b/api/common/serializers.py index 272d0d8..e034254 100644 --- a/api/common/serializers.py +++ b/api/common/serializers.py @@ -49,9 +49,14 @@ def get_enums(self, obj): `FieldsMixin` cannot prune inside it on its own. This reads the branch under "enums" and filters by hand instead. `MetaView` still names every enum in the one place it always has. + + `reject_unknown` is the same refusal `_prune` makes. Without it + a typed enum name answered `200 {"enums": {}}`, which is the + empty payload the 400 exists to prevent. """ enums = obj["enums"] wanted = self.child_tree("enums") if not wanted: return enums + self.reject_unknown(wanted, enums) return {name: labels for name, labels in enums.items() if name in wanted} diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index 74f6630..fbc8ed2 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -291,6 +291,36 @@ def test_a_path_carries_enough_to_link_to_each_step(): assert ComponentSerializer(overall).data["path"] == [] +@pytest.mark.django_db +def test_fields_prunes_inside_a_path_step(): + # `path` is a method field, so `_prune` cannot reach the step + # serializer. A dotted path into it was ignored, and the contract + # promises a dotted path prunes wherever it is accepted. + service = ServiceFactory(name="Twilio") + overall = ComponentFactory(service=service, name="Twilio", is_overall=True) + child = ComponentFactory(service=service, name="SMS", parent=overall) + + url = reverse("component-detail", args=[child.id]) + body = APIClient().get(url, {"fields": "path.name"}).json() + + assert body["path"] == [{"name": "Twilio"}] + + +@pytest.mark.django_db +def test_fields_refuses_an_unknown_name_under_a_path_step(): + # Nothing pruned and nothing refused. A typo read as a step that + # holds every field, which is the opposite of what was asked for. + service = ServiceFactory(name="Twilio") + overall = ComponentFactory(service=service, name="Twilio", is_overall=True) + child = ComponentFactory(service=service, name="SMS", parent=overall) + + url = reverse("component-detail", args=[child.id]) + response = APIClient().get(url, {"fields": "path.nonsense"}) + + assert response.status_code == 400 + assert response.json() == {"fields": ["Unknown field: nonsense."]} + + @pytest.mark.django_db def test_a_finished_maintenance_window_is_not_live(): # A provider often leaves the phase behind when a window ends, so diff --git a/api/tests/test_meta.py b/api/tests/test_meta.py index 7f64536..1e179b7 100644 --- a/api/tests/test_meta.py +++ b/api/tests/test_meta.py @@ -96,6 +96,16 @@ def test_fields_keeps_two_dotted_paths_under_enums(): assert set(body["enums"]) == {"status_source", "event_kind"} +@pytest.mark.django_db +def test_fields_refuses_an_unknown_name_under_enums(): + # `enums` is a plain dict, filtered by hand, so `_prune` never saw + # this name. It answered `200 {"enums": {}}`, which is the silent + # empty payload the 400 exists to prevent. + response = APIClient().get(reverse("meta") + "?fields=enums.nonsense") + assert response.status_code == 400 + assert response.json() == {"fields": ["Unknown field: nonsense."]} + + @pytest.mark.django_db def test_meta_publishes_the_deployment_defaults(): body = APIClient().get(reverse("meta")).json() From 5befea93f82189cfa8fa261d48edcea3b73495dd Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 22:44:25 -0700 Subject: [PATCH 29/94] fix: the derived columns are rebuilt by the model, not by the admin `rebuild_ancestry` and `rebuild_search` ran from the admin and from a poll. Every other writer left the columns stale: a shell session, a data migration, a later API write. The symptom is silence. The component is absent from `?ancestor=`, from `?q=` and from every `descendant_count`, and an untracked service is never polled to repair it. `save` now rebuilds when the row is created, or when its name, parent or service changed. Reconcile holds that back through `held_rebuilds` and keeps its single rebuild at the end. Without the hold, a poll of a hundred components would rebuild a hundred rows a hundred times. The admin also rebuilt only the component's new service, so moving one between services left the old service naming it. The hook rebuilds both, and the ancestry walk now stops at a parent outside the service. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 19 -------- api/catalog/models.py | 80 +++++++++++++++++++++++++++++++- api/polling/reconcile.py | 67 ++++++++++++++------------ api/tests/test_admin.py | 10 ++-- api/tests/test_catalog_models.py | 53 +++++++++++++++++++++ api/tests/test_reconcile.py | 25 ++++++++++ 6 files changed, 200 insertions(+), 54 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index 631d409..d9a5391 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -35,7 +35,6 @@ from common.queries import related_count from polling.importer import import_from_url from polling.models import Poller, PollRun -from polling.reconcile import rebuild_ancestry, rebuild_search from status.choices import EventKind, Severity from status.models import ComponentStatus, ServiceEvent from status.queries import CURRENT_SEVERITY @@ -583,24 +582,6 @@ class ServiceComponentAdmin( # acts on a single object, which has one state to flip. actions = ["feature_selected", "unfeature_selected"] - def save_model(self, request, obj, form, change): - """Keep the derived columns true after an edit by hand. - - `rebuild_ancestry` and `rebuild_search` run from reconcile only. - A component added or reparented here held no ancestry and no - search vector. It was then invisible to `?ancestor=`, counted by - no `descendant_count`, and unfindable by `?q=`. - - A poll repairs a tracked service. An untracked or paused one is - never selected, so the staleness would be permanent. - - Here and not on `ServiceComponent.save`, because reconcile writes - these rows with `bulk_update` and would pay the pass per row. - """ - super().save_model(request, obj, form, change) - rebuild_ancestry(obj.service) - rebuild_search(obj.service) - def get_form(self, request, obj=None, **kwargs): # The parent picker needs to know which service is asking. request._editing = obj diff --git a/api/catalog/models.py b/api/catalog/models.py index 5e7d0fc..ace0b94 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -1,3 +1,5 @@ +from contextlib import contextmanager +from contextvars import ContextVar from urllib.parse import urlparse, urlunparse from django.contrib.postgres.fields import ArrayField @@ -257,6 +259,30 @@ def _descendant_count(): return Coalesce(models.Subquery(counted, output_field=models.IntegerField()), 0) +_rebuilds_held = ContextVar("catalog_rebuilds_held", default=False) + + +@contextmanager +def held_rebuilds(): + """Hold back the per-save rebuild, for a caller doing its own. + + `ServiceComponent.save` rewrites the whole service. A caller that + saves many components of one service pays that per row. A poll of a + hundred components then costs a hundred passes over a hundred rows. + + A caller that holds the rebuild owes the service one afterwards. + `polling.reconcile._upsert_components` is the only one. + + A `ContextVar` rather than a flag, so one worker thread cannot hold + back another's rebuild. + """ + token = _rebuilds_held.set(True) + try: + yield + finally: + _rebuilds_held.reset(token) + + class ServiceComponentQuerySet(models.QuerySet): def visible(self): """The components a caller is served, anywhere. @@ -441,12 +467,24 @@ def is_tracked_by(self, user): return self.boards.filter(owner=user).exists() return prepared + # `ancestor_ids` and `search_document` are built from these. A save + # touching none of them leaves both true. + REBUILD_TRIGGERS = frozenset( + {"name", "parent", "parent_id", "service", "service_id"} + ) + def save(self, *args, **kwargs): - """Keep the archive date in step with the flag. + """Keep the archive date in step with the flag, and the tree true. A bulk update does not come through here, so anything writing the flag that way sets the date itself. The constraint is what makes that a failure rather than a quiet disagreement. + + The rebuild was on the admin alone. Every other writer left the + derived columns stale. The symptom is silence: the component is + absent from `?ancestor=`, from `?q=` and from every + `descendant_count`. A poll repairs a tracked service, so a + paused or untracked one stayed wrong forever. """ if self.is_archived and self.archived_at is None: self.archived_at = timezone.now() @@ -455,7 +493,47 @@ def save(self, *args, **kwargs): fields = kwargs.get("update_fields") if fields is not None and "is_archived" in fields: kwargs["update_fields"] = [*fields, "archived_at"] + stale = self._services_to_rebuild(kwargs.get("update_fields")) super().save(*args, **kwargs) + if stale: + from polling.reconcile import rebuild_ancestry, rebuild_search + + for service in stale: + rebuild_ancestry(service) + rebuild_search(service) + + def _services_to_rebuild(self, update_fields): + """Whose derived columns this save is about to invalidate. + + Empty is the normal answer. A poll saves every row of a service + and usually moves none of them. So the check costs no query when + nothing the columns are built from changed. + + A create counts. A component made in a shell otherwise holds no + ancestry and no search vector. + + Call this before the write. It reads the row as it still stands. + """ + if _rebuilds_held.get(): + return () + if self._state.adding: + return (self.service,) + if update_fields is not None and self.REBUILD_TRIGGERS.isdisjoint( + update_fields + ): + return () + was = ( + ServiceComponent.objects.filter(pk=self.pk) + .values_list("name", "parent_id", "service_id") + .first() + ) + if was is None or was == (self.name, self.parent_id, self.service_id): + return () + if was[2] == self.service_id: + return (self.service,) + # A move between services leaves the old one still naming this + # row in its `ancestor_ids` and in its search documents. + return (self.service, Service.objects.get(pk=was[2])) def clean(self): """A component sits under one of its own service's components. diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index ff96270..3c90877 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -4,7 +4,7 @@ from django.db.models import Value from django.utils import timezone -from catalog.models import ServiceComponent +from catalog.models import ServiceComponent, held_rebuilds from polling.system_events import claim, reconcile_system_events from status.models import ComponentStatus, EventUpdate, ServiceEvent @@ -56,33 +56,37 @@ def _signed(author, **fields): def _upsert_components(service, components, author): rows = {} - for incoming in components: - row, _ = ServiceComponent.objects.update_or_create( - service=service, - external_id=incoming.external_id, - **_signed( - author, - name=incoming.name, - status_page_order=incoming.order, - is_overall=incoming.is_overall, - is_archived=False, - archived_at=None, - ), - ) - rows[incoming.external_id] = row - - # Parents in a second pass: a child may arrive before its parent exists. - for incoming in components: - parent = ( - rows.get(incoming.parent_external_id) - if incoming.parent_external_id - else None - ) - row = rows[incoming.external_id] - if row.parent_id != (parent.id if parent else None): - row.parent = parent - row.updated_by = author - row.save(update_fields=["parent", "updated_by"]) + # `ServiceComponent.save` rebuilds the whole service. This pass + # saves every component of it, twice over, so the hook would make + # one poll quadratic. It runs once at the end instead. + with held_rebuilds(): + for incoming in components: + row, _ = ServiceComponent.objects.update_or_create( + service=service, + external_id=incoming.external_id, + **_signed( + author, + name=incoming.name, + status_page_order=incoming.order, + is_overall=incoming.is_overall, + is_archived=False, + archived_at=None, + ), + ) + rows[incoming.external_id] = row + + # Parents second: a child may arrive before its parent exists. + for incoming in components: + parent = ( + rows.get(incoming.parent_external_id) + if incoming.parent_external_id + else None + ) + row = rows[incoming.external_id] + if row.parent_id != (parent.id if parent else None): + row.parent = parent + row.updated_by = author + row.save(update_fields=["parent", "updated_by"]) rebuild_ancestry(service) rebuild_search(service) return rows @@ -103,8 +107,13 @@ def rebuild_ancestry(service): def chain(node): # The column points at its own table, so the data allows a loop. # `seen` ends the walk. Without it bad data hangs the worker. + # + # `up in parents` keeps the chain inside this service. Moving + # a component to another service leaves its old children + # pointing across. A step there names a row this service's + # lists never hold. walked, seen, up = [], {node}, parents.get(node) - while up is not None and up not in seen: + while up is not None and up not in seen and up in parents: seen.add(up) walked.append(up) up = parents.get(up) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index d034e43..a2d8854 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -920,12 +920,12 @@ def component_form_data(staff_client, component, **fields): @pytest.mark.django_db def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): - """The admin is the only writer outside reconcile. + """The admin posts a form, so it reaches the model by another road. - `rebuild_ancestry` and `rebuild_search` run on a poll. A component - reparented here held no ancestry and no search vector. It was then - invisible to `?ancestor=` and unfindable by `?q=`. An untracked - service is never polled, so nothing would repair it. + `ServiceComponent.save` rebuilds the derived columns. A component + reparented here must come out of it with ancestry and a search + vector. Without them it is invisible to `?ancestor=` and unfindable + by `?q=`. An untracked service is never polled to repair it. """ from catalog.models import ServiceComponent from tests.factories import ComponentFactory, ServiceFactory diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index fe8a369..e7dc48d 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -159,6 +159,59 @@ def test_the_flag_and_the_date_cannot_disagree(): ServiceComponent.objects.filter(pk=component.pk).update(is_archived=True) +def test_a_reparent_in_plain_python_rebuilds_the_derived_columns(db): + # An admin-only hook left every other writer stale: a shell session, + # a data migration, a later API write. The symptom is invisible. The + # row is missing from `?ancestor=`, from `?q=` and from every count. + service = ServiceFactory(name="Twilio") + parent = ComponentFactory(service=service, name="Programmable Messaging") + child = ComponentFactory(service=service, name="SMS") + + child.parent = parent + child.save() + + child.refresh_from_db() + assert child.ancestor_ids == [parent.pk] + assert list(ServiceComponent.objects.search("programmable sms")) == [child] + + +def test_a_move_between_services_rebuilds_the_service_left_behind(db): + # Only the new service was rebuilt. The old one kept rows naming + # this component in `ancestor_ids` and in their search documents. + old = ServiceFactory(name="Twilio") + new = ServiceFactory(name="Vonage") + top = ComponentFactory(service=old, name="Programmable Messaging") + leaf = ComponentFactory(service=old, name="SMS", parent=top) + rebuild_ancestry(old) + rebuild_search(old) + assert list(ServiceComponent.objects.search("programmable sms")) == [leaf] + + top.parent = None + top.service = new + top.save() + + leaf.refresh_from_db() + assert leaf.ancestor_ids == [] + assert list(ServiceComponent.objects.search("programmable sms")) == [] + + +def test_a_save_that_moves_nothing_rebuilds_nothing(db, monkeypatch): + # A rebuild rewrites every row of the service. A save of an + # unrelated column must not pay for it, or one poll goes quadratic. + from polling import reconcile + + component = ComponentFactory() + calls = [] + monkeypatch.setattr( + reconcile, "rebuild_search", lambda service: calls.append(service) + ) + + component.is_featured = True + component.save() + + assert calls == [] + + def test_ancestor_ids_are_stored_top_down(db): # A descendant query reads this array. Walking `parent` per row # cannot use an index, and `for_display` only selects two levels up. diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index 912ff15..b0e87ac 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -314,6 +314,31 @@ def test_a_poll_writes_the_ancestor_chain(): assert child.ancestor_ids == [top.id] +@pytest.mark.django_db +def test_a_poll_rebuilds_the_service_once_not_once_a_component(monkeypatch): + # `ServiceComponent.save` rebuilds the whole service, and this pass + # saves every component of it. Unsuppressed, a hundred components + # cost a hundred rebuilds of a hundred rows, on every poll. + from polling import reconcile + + calls = [] + real = reconcile.rebuild_ancestry + monkeypatch.setattr( + reconcile, + "rebuild_ancestry", + lambda service: calls.append(service) or real(service), + ) + + service = ServiceFactory() + rows = [_component(external_id=str(n)) for n in range(5)] + apply_fetch(service, rows, [], StatusSource.PROVIDER) + assert len(calls) == 1 + + calls.clear() + apply_fetch(service, rows, [], StatusSource.PROVIDER) + assert len(calls) == 1 + + @pytest.mark.django_db def test_a_reparent_rewrites_the_subtree(): # The chain of a grandchild changes when its parent moves, and From b1339ce81f0ea48fa2cd771061cc95bedd0ce733 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 22:47:57 -0700 Subject: [PATCH 30/94] feat: any component can be featured, from its own change page Featuring was restricted to a service's overall component. The bulk actions filtered on `is_overall` and reported a skip, and the field was absent from the change form. `suggested` reads the flag on every component, because Discover lists every component. Featuring one part of a service is an editorial choice, not an accident to guard against. The change page now carries the flag, and neither bulk action refuses a leaf. The service page no longer offers featuring at all: one flag with three writers and one restriction between them was the reason the restriction had to exist. Featuring a service means ticking its overall component, which the component changelist already filters for. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 80 ++++++++++++----------------------- api/tests/test_admin.py | 60 ++++++++++++-------------- api/tests/test_catalog_api.py | 18 ++++++++ 3 files changed, 71 insertions(+), 87 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index d9a5391..9de35f7 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -304,35 +304,6 @@ def queryset(self, request, queryset): ) -class OverallComponentInline(StackedInline): - """Featuring lives on the rollup, and an admin asks about the service. - - `is_featured` is the first key of the suggested sort. Ticking it on - a service's overall component is what surfaces that service, so - that is the question this answers. - - `ServiceComponentInline` also edits `ServiceComponent`, through the - same FK. `ModelAdmin._create_formsets` auto-suffixes a repeated - prefix, so the two never collide even though both derive the same - default one. - """ - - model = ServiceComponent - fields = ["is_featured"] - extra = 0 - max_num = 1 - can_delete = False - verbose_name = "Featuring" - verbose_name_plural = "Featuring" - - def get_queryset(self, request): - return super().get_queryset(request).filter(is_overall=True) - - def has_add_permission(self, request, obj): - # Reconcile makes the rollup. An admin ticks its flag. - return False - - @admin.register(Service) class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): list_display = [ @@ -369,7 +340,6 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): # What the service is, then what is happening to it. Then how it # is configured, then the log of us reading it. inlines = [ - OverallComponentInline, ServiceComponentInline, ServiceEventInline, StatusPageInline, @@ -526,8 +496,9 @@ class ServiceComponentAdmin( "display_archived", "display_related", "watchers", - # Only the overall component's flag surfaces its service. The - # flag is real on any row, but only that one row does anything. + # `suggested` reads this flag on every component. Ticking it on + # a rollup surfaces the service. Ticking it on a leaf surfaces + # that part. "is_featured", ] search_fields = [ @@ -573,6 +544,16 @@ class ServiceComponentAdmin( _("Position"), {"fields": ["parent", "status_page_order", "is_overall"]}, ), + ( + _("Featuring"), + { + "fields": ["is_featured"], + "description": _( + "Leads the suggested sort on the catalog. Tick it on a " + "service's overall component to surface the service." + ), + }, + ), audit_section(), ] inlines = [ComponentStatusInline] @@ -668,40 +649,33 @@ def display_severity(self, obj): def watchers(self, obj): return obj.watcher_count - def _skip_note(self, queryset): - """How many of the selection the action would not touch, and why. - - `suggested` sorts on `-is_featured` first, over every component, - rollups included. Featuring a leaf would win Discover outright, - which nothing intends. Only the overall component may hold the - flag, so a silent skip would look like nothing happened. - """ - skipped = queryset.exclude(is_overall=True).count() - if not skipped: - return "" - return " " + _( - "%(skipped)d selected component(s) skipped: not an overall component." - ) % {"skipped": skipped} - @admin.action(description=_("Feature selected components")) def feature_selected(self, request, queryset): + """Feature every selected component, rollup or leaf. + + The change form above sets the same flag one row at a time. + Both are unrestricted, because `suggested` reads the flag on + every component and a featured leaf is a real editorial choice. + """ count = 0 - for component in queryset.filter(is_overall=True, is_featured=False): + for component in queryset.filter(is_featured=False): component.is_featured = True component.save(update_fields=["is_featured"]) count += 1 - message = _("Featured %(count)d component(s).") % {"count": count} - self.message_user(request, message + self._skip_note(queryset)) + self.message_user( + request, _("Featured %(count)d component(s).") % {"count": count} + ) @admin.action(description=_("Unfeature selected components")) def unfeature_selected(self, request, queryset): count = 0 - for component in queryset.filter(is_overall=True, is_featured=True): + for component in queryset.filter(is_featured=True): component.is_featured = False component.save(update_fields=["is_featured"]) count += 1 - message = _("Unfeatured %(count)d component(s).") % {"count": count} - self.message_user(request, message + self._skip_note(queryset)) + self.message_user( + request, _("Unfeatured %(count)d component(s).") % {"count": count} + ) @admin.register(ServiceRequest) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index a2d8854..835810a 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -625,21 +625,6 @@ def test_a_person_cannot_open_somebody_elses_board(): mine.full_clean() -@pytest.mark.django_db -def test_the_service_admin_features_through_the_overall_component(admin_client, db): - # is_featured moved to the component. An admin asks "feature this - # service", so the service page has to answer it. - from tests.factories import ComponentFactory, ServiceFactory - - service = ServiceFactory() - ComponentFactory(service=service, is_overall=True) - response = admin_client.get( - reverse("admin:catalog_service_change", args=[service.pk]) - ) - assert response.status_code == 200 - assert b"is_featured" in response.content - - @pytest.mark.django_db def test_the_component_list_shows_watchers(admin_client, db): # The count is an annotation. A column that forgets to annotate it @@ -695,7 +680,7 @@ def test_the_feature_action_flips_the_flag(admin_client, db): @pytest.mark.django_db def test_the_unfeature_action_flips_it_back(admin_client, db): # The reverse has to exist too. Otherwise featuring by mistake is - # permanent, short of editing the row through the service's inline. + # permanent, short of editing the row on its change page. from tests.factories import ComponentFactory component = ComponentFactory(is_featured=True, is_overall=True) @@ -708,48 +693,55 @@ def test_the_unfeature_action_flips_it_back(admin_client, db): @pytest.mark.django_db -def test_the_feature_action_skips_a_non_overall_component(admin_client, db): - # `suggested` sorts every component on `-is_featured` first, rollups - # included. Featuring a leaf would win Discover outright. Nothing in - # the design intends that, so the action must refuse it and say why. +def test_the_feature_action_features_a_leaf(admin_client, db): + # Discover lists every component, and `suggested` reads the flag on + # every row. Featuring one part of a service is an editorial choice + # an admin is allowed to make. from tests.factories import ComponentFactory leaf = ComponentFactory(is_featured=False, is_overall=False) - response = admin_client.post( + admin_client.post( reverse("admin:catalog_servicecomponent_changelist"), {"action": "feature_selected", "_selected_action": [str(leaf.pk)]}, - follow=True, ) leaf.refresh_from_db() - assert leaf.is_featured is False - assert b"skipped" in response.content + assert leaf.is_featured is True @pytest.mark.django_db -def test_a_mixed_selection_features_the_overall_one_and_skips_the_leaf( - admin_client, db -): - # A selection an admin makes by hand is not guaranteed to be clean. - # The overall row still gets featured; the leaf does not, silently - # or otherwise. +def test_the_feature_action_takes_a_mixed_selection_whole(admin_client, db): + # A rollup and a leaf are both featurable, so a selection holding + # one of each leaves nothing behind. from tests.factories import ComponentFactory, ServiceFactory service = ServiceFactory() overall = ComponentFactory(service=service, is_overall=True, is_featured=False) leaf = ComponentFactory(service=service, is_overall=False, is_featured=False) - response = admin_client.post( + admin_client.post( reverse("admin:catalog_servicecomponent_changelist"), { "action": "feature_selected", "_selected_action": [str(overall.pk), str(leaf.pk)], }, - follow=True, ) overall.refresh_from_db() leaf.refresh_from_db() assert overall.is_featured is True - assert leaf.is_featured is False - assert b"skipped" in response.content + assert leaf.is_featured is True + + +@pytest.mark.django_db +def test_the_component_form_features_a_leaf(staff_client): + # The change page is where an admin already is when they decide to + # feature something. Without the field the form cannot set it. + from tests.factories import ComponentFactory + + leaf = ComponentFactory(is_overall=False, is_featured=False) + url, data = component_form_data(staff_client, leaf, is_featured="on") + assert staff_client.post(url, data).status_code == 302 + + leaf.refresh_from_db() + assert leaf.is_featured is True @pytest.mark.django_db diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index fbc8ed2..34470bd 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -249,6 +249,24 @@ def test_a_featured_component_still_leads(): assert names.index("Featured") < names.index("Plain") +@pytest.mark.django_db +def test_a_featured_leaf_leads_as_well_as_a_featured_rollup(): + # Discover lists every component, and the sort reads the flag on + # every row. Featuring one part of a service surfaces that part. + service = ServiceFactory(name="Twilio") + StatusPageFactory(service=service) + _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True, external_id="roll") + leaf = _with_status(service, Severity.OPERATIONAL, external_id="leaf") + leaf.name = "SMS" + leaf.is_featured = True + leaf.save(update_fields=["name", "is_featured"]) + + names = [ + r["name"] for r in APIClient().get(reverse("component-list")).json()["results"] + ] + assert names[0] == "SMS" + + @pytest.mark.django_db def test_a_component_with_no_reading_sorts_last_not_first(): # Never seen is not the same as healthy, and it must not pose as From 447feceed0f619ec157c549d8756e3a3c12f47f6 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 22:50:41 -0700 Subject: [PATCH 31/94] docs: four corrections, and a test for the archived rollup carve-out Nothing pinned the documented exception that a service whose rollup is archived still returns `overall_component`. Moving `visible` into `for_display` is the obvious simplification, and it would have deleted that guarantee with a green suite. Both readers now carry a test. The ranked paging test claimed the `-created_at` tiebreak enters the cursor. `CursorPagination` builds its position from the first ordering term alone, so it never does. The test pins that paging a ranked search returns each row once, which is what its docstring now says. Its loop takes a page ceiling, so a repeat fails rather than hangs. The contract said query parameters are declared in a `FilterSet` and counted `q` among them. `q` is ranked in the view. It also said two collections where the board list is a third. The justfile claimed `just check` runs every check CI runs. CI also runs a Trivy scan that fails a merge on a HIGH or CRITICAL CVE. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_catalog_api.py | 19 +++++++++++++++++++ api/tests/test_component_api.py | 12 ++++++++---- bin/tests/test_merge_gate.py | 6 +++++- docs/api/openapi.yaml | 8 +++++--- justfile | 8 ++++++-- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index 34470bd..bdc6253 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -124,6 +124,25 @@ def test_the_overall_component_is_a_plain_component(): assert field in overall, f"{field} missing from the overall component" +@pytest.mark.django_db +def test_a_service_whose_rollup_is_archived_still_has_a_header_status(): + # The one documented exception to the archived rule. Moving + # `visible` into `for_display` is the obvious simplification, and it + # would leave the service page with no status at all. + from catalog.models import Service + + service = ServiceFactory() + StatusPageFactory(service=service) + rollup = _with_status(service, Severity.MAJOR_OUTAGE, is_overall=True) + rollup.is_archived = True + rollup.save(update_fields=["is_archived"]) + + body = _service_detail(service) + assert body["overall_component"]["id"] == str(rollup.id) + assert body["overall_component"]["archived_at"] is not None + assert Service.objects.get(pk=service.pk).overall_component() == rollup + + @pytest.mark.django_db def test_severity_filters_use_the_orm_path(): # The contract's name is not a column. It points at an annotation, diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py index 950c1a1..17afaee 100644 --- a/api/tests/test_component_api.py +++ b/api/tests/test_component_api.py @@ -196,9 +196,9 @@ def test_event_narrows_to_the_components_one_event_affects(client, tree): def test_a_ranked_search_pages_without_repeating_a_row(client): """`?q=` is Discover's default path, and `rank` ties heavily. - Identical documents rank identically. `EnvelopePagination` appends - `-created_at` to make the key unique. Without it a cursor repeats or - skips a row, and infinite scroll shows it. + Identical documents rank identically. A page boundary inside a run + of ties is where a cursor repeats a row or skips one. Infinite + scroll then shows the fault. """ service = ServiceFactory(name="Twilio") for _ in range(7): @@ -206,10 +206,14 @@ def test_a_ranked_search_pages_without_repeating_a_row(client): rebuild_ancestry(service) rebuild_search(service) - seen = [] + seen, pages = [], 0 url = reverse("component-list") params = {"q": "twilio sms", "page_size": 2} while url: + pages += 1 + # Seven rows, two a page. A cursor that repeats a page would + # otherwise hang the suite rather than fail it. + assert pages <= 4, "paging did not reach the end" body = client.get(url, params).json() seen += [row["id"] for row in body["results"]] url, params = body["next"], {} diff --git a/bin/tests/test_merge_gate.py b/bin/tests/test_merge_gate.py index 7e949f1..ad7e2a0 100644 --- a/bin/tests/test_merge_gate.py +++ b/bin/tests/test_merge_gate.py @@ -1,4 +1,8 @@ -"""`just check` and the CI workflow must gate on the same checks. +"""Every check `just check` runs must also run in CI. + +Not the reverse. CI also scans the image for CVEs, which needs the +network and a vulnerability database. So a green `just check` is not a +green merge gate, and the justfile says so. `bin/` has no dependency manager and no Django. These tests use only the standard library, so they run on a machine that has done nothing but diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index cc562d2..e4e604c 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -86,7 +86,8 @@ info: layers that kind of value onto its own list row. Query parameters are declared in a `FilterSet`. Almost none is generated from a column of - the same name. `is_overall` and `kind` are the two that are. + the same name. `is_overall` and `kind` are the two that are. `q` is no filter at all. + `ComponentListView.filter_queryset` ranks on it before the backends run. So a parameter is not an ORM path, and a field's path in the response body does not build its filter. `service` takes a slug where the body carries a nested `ServiceRef`. `component` @@ -96,8 +97,9 @@ info: Read the operation's own parameter list instead. It says what each one takes and what it narrows. - Eleven are declared across the two collections: `q`, `service`, `ancestor`, `event`, - `component`, `dashboard`, `is_tracked`, `phase`, and the three `status__severity` forms. + Eleven narrow the three collections: `q`, `service`, `ancestor`, `event`, `component`, + `dashboard`, `is_tracked`, `phase`, and the three `status__severity` forms. The board list + takes those three severity forms and nothing else. Two ordering values are declared as well, `suggested` and `next_transition`. Neither names a column. A view annotates the fields each one sorts on. diff --git a/justfile b/justfile index ee881ec..43b2c54 100644 --- a/justfile +++ b/justfile @@ -103,9 +103,13 @@ test-bin: # image rather than the host. Run it before opening a pull request. # The two stdlib-only checks run on the host, because a container costs # more than it saves. `bin/tests/test_merge_gate.py` holds the two lists -# together. +# together, in one direction: everything here also runs in CI. +# +# CI runs one more. It scans the image with Trivy for HIGH and CRITICAL +# CVEs and fails the merge on a hit. That needs the network and a +# vulnerability database, so it stays out of this recipe. -# Every check CI runs, each where CI runs it. +# Every CI check but the image scan, each where CI runs it. check: image docker compose -f docker-compose.test.yml run --rm pytest -q -n auto --cov-fail-under=85 docker compose -f docker-compose.test.yml run --rm ruff From c352304dd023c2425aa7484dcacbeb5b52a2cdb0 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Thu, 3 Sep 2026 23:01:24 -0700 Subject: [PATCH 32/94] fix: check_prose resolves its default scan target from __file__ The default target was pathlib.Path("api"), resolved against the process's working directory. A hand run from api/ then looked for api/api/, found a handful of settings files, and reported clean. A check that reports success without checking is worse than no check: it is silent where a missing check is at least conspicuous. An implementer verified "prose clean" twice from inside api/, and the real pre-commit hook then caught two genuine over-length sentences. The default now resolves from __file__, so it is the same regardless of cwd. It widens to cover bin/ as well as api/, since the script's own prose was otherwise unchecked by its own default and bin/ is already clean. A run that examines zero files, whether from a missing default target or an explicit path matching nothing, now exits non-zero with a message, instead of reporting success on nothing. A clean run now names what it scanned, so a false pass is not silent. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- bin/check_prose.py | 50 ++++++++++++++++++++- bin/tests/test_check_prose.py | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/bin/check_prose.py b/bin/check_prose.py index b750f9d..2497bd3 100755 --- a/bin/check_prose.py +++ b/bin/check_prose.py @@ -31,6 +31,10 @@ # A code span, however many tokens inside, is one thing to read. CODE_SPAN = re.compile(r"`[^`]*`") +# The repository root, from this file's own location. A hand run from +# `api/` or `bin/` then resolves the same default as one from the root. +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent + def sentences(blocks): """Split prose into sentences, keeping the line each one started on.""" @@ -169,25 +173,67 @@ def prose_problems(blocks): return problems +def default_targets(): + """The directories scanned when no path is given: api/ and bin/. + + A stdlib script checks its own prose too, not only Django's. + """ + return [REPO_ROOT / "api", REPO_ROOT / "bin"] + + +def default_paths(): + """Every `.py` file under a default target that exists.""" + return sorted( + path + for target in default_targets() + if target.is_dir() + for path in target.rglob("*.py") + ) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("paths", nargs="*", type=pathlib.Path) parser.add_argument("--limit", type=int, default=8) args = parser.parse_args() - paths = args.paths or sorted(pathlib.Path("api").rglob("*.py")) + if args.paths: + paths = args.paths + label = ", ".join(str(p) for p in paths) + else: + targets = default_targets() + if not any(target.is_dir() for target in targets): + existing = ", ".join(str(t) for t in targets) + print(f"error: no scan target exists: {existing}", file=sys.stderr) + return 1 + paths = default_paths() + label = " and ".join( + str(t.relative_to(REPO_ROOT)) for t in targets if t.is_dir() + ) + total = 0 + examined = 0 for path in paths: if path.suffix != ".py" or not path.is_file(): continue if any(part in {".venv", "__pycache__", "migrations"} for part in path.parts): continue + examined += 1 for line, rule, detail in check(path, args.limit): print(f"{path}:{line}: {rule}: {detail}") total += 1 + + # A checker that examines nothing has not passed. It has not run. + if examined == 0: + print(f"error: no files examined in {label}.", file=sys.stderr) + return 1 if total: print(f"\n{total} problems. See AGENTS.md.", file=sys.stderr) - return 1 if total else 0 + return 1 + + noun = "file" if examined == 1 else "files" + print(f"Checked {examined} {noun} in {label}. Clean.") + return 0 if __name__ == "__main__": diff --git a/bin/tests/test_check_prose.py b/bin/tests/test_check_prose.py index 702da9b..c0d3708 100644 --- a/bin/tests/test_check_prose.py +++ b/bin/tests/test_check_prose.py @@ -5,13 +5,19 @@ `just init`. Run them with `python3 -m unittest discover -s bin/tests`. """ +import io +import os +import subprocess import sys import unittest +from contextlib import redirect_stderr, redirect_stdout from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import check_prose +SCRIPT = Path(check_prose.__file__).resolve() + class WordsTests(unittest.TestCase): def test_a_multi_word_code_span_counts_as_one_word(self): @@ -58,5 +64,84 @@ def test_a_twenty_one_word_sentence_is_flagged(self): self.assertEqual([rule for _line, rule, _detail in problems], ["long"]) +class DefaultTargetTests(unittest.TestCase): + def test_default_paths_are_the_same_from_the_repository_root_and_a_subdirectory( + self, + ): + # The default used to resolve "api" against the caller's cwd. + # A run from api/ then scanned api/api/, a handful of settings + # files, instead of api/. The default must not depend on cwd. + cwd = os.getcwd() + try: + os.chdir(check_prose.REPO_ROOT) + from_root = check_prose.default_paths() + os.chdir(check_prose.REPO_ROOT / "api") + from_subdir = check_prose.default_paths() + finally: + os.chdir(cwd) + self.assertEqual(from_root, from_subdir) + self.assertTrue(from_root, "the default target found no files at all") + + def test_a_missing_default_target_is_not_silently_skipped(self): + # A target that used to hold Python and no longer does must + # fail loudly. It must not report a clean run over nothing. + real_targets = check_prose.default_targets + real_argv = sys.argv + check_prose.default_targets = lambda: [check_prose.REPO_ROOT / "does-not-exist"] + sys.argv = ["check_prose.py"] + out, err = io.StringIO(), io.StringIO() + try: + with redirect_stdout(out), redirect_stderr(err): + status = check_prose.main() + finally: + check_prose.default_targets = real_targets + sys.argv = real_argv + self.assertNotEqual(status, 0) + self.assertIn("no scan target exists", err.getvalue()) + + +class MainExitStatusTests(unittest.TestCase): + """Run the real script as a subprocess: main()'s exit code is the contract.""" + + def run_script(self, *args, cwd=None): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + cwd=cwd, + check=False, + ) + + def test_a_clean_default_run_exits_zero_and_names_what_it_scanned(self): + result = self.run_script(cwd=str(check_prose.REPO_ROOT)) + self.assertEqual(result.returncode, 0, result.stderr) + # Silence on success is what let two runs from api/ report clean + # while checking a handful of settings files. The count must show. + self.assertIn("Checked", result.stdout) + self.assertIn("api", result.stdout) + self.assertIn("bin", result.stdout) + + def test_the_default_run_is_identical_from_the_repository_root_and_bin(self): + from_root = self.run_script(cwd=str(check_prose.REPO_ROOT)) + from_bin = self.run_script(cwd=str(check_prose.REPO_ROOT / "bin")) + self.assertEqual(from_root.returncode, 0, from_root.stderr) + self.assertEqual(from_root.stdout, from_bin.stdout) + + def test_an_explicit_path_matching_no_file_exits_non_zero(self): + # A caller who passes a path that matches nothing has the same + # silent-pass problem as a broken default. Treated the same way. + result = self.run_script("does/not/exist.py", cwd=str(check_prose.REPO_ROOT)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("no files examined", result.stderr) + + def test_an_explicit_directory_with_no_python_files_exits_non_zero(self): + # check() only ever receives files listed directly in args.paths, + # so a directory argument matches zero files. That must error, + # not report a clean run. + result = self.run_script("docs", cwd=str(check_prose.REPO_ROOT)) + self.assertNotEqual(result.returncode, 0) + self.assertIn("no files examined", result.stderr) + + if __name__ == "__main__": unittest.main() From 7ae0210ad7b78e75789e663531816c4061454b78 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 11:23:57 -0700 Subject: [PATCH 33/94] feat: ancestry is a table of links, not an array of ids `ServiceComponent.ancestor_ids` bought an indexed containment test and held no reference. `parent` is `SET_NULL`, so nothing cleaned the array when a component went. Every descendant kept naming a row that was gone, and only a poll of that service repaired it. An untracked service is never polled again. `rebuild_search` papered over the dangling ids with a default empty name. `ComponentAncestor` makes both sides foreign keys. A delete cascades in the same statement, so no reader can be served a step that names nothing. The unique pair indexes the descendant direction, which is what `?ancestor=` and every descendant count ask for. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/filters.py | 10 +- .../0008_component_ancestor_table.py | 123 ++++++++++++++++ api/catalog/models.py | 132 ++++++++++++------ api/polling/reconcile.py | 65 ++++++--- api/tests/factories.py | 13 ++ api/tests/test_admin.py | 4 +- api/tests/test_catalog_models.py | 51 +++++-- api/tests/test_reconcile.py | 17 +-- 8 files changed, 324 insertions(+), 91 deletions(-) create mode 100644 api/catalog/migrations/0008_component_ancestor_table.py diff --git a/api/catalog/filters.py b/api/catalog/filters.py index c5eee74..5d68d4c 100644 --- a/api/catalog/filters.py +++ b/api/catalog/filters.py @@ -15,7 +15,7 @@ class ComponentFilter(SeverityFilterMixin): service = filters.CharFilter(field_name="service__slug") # Every descendant, not one level. `parent` would name a query this # does not run. - ancestor = filters.UUIDFilter(method="filter_ancestor") + ancestor = filters.UUIDFilter(field_name="ancestor_links__ancestor") event = filters.UUIDFilter(field_name="events__id") # `for_display` annotates this per viewer. It is not a column, so # there is nothing to generate the filter from. @@ -25,14 +25,6 @@ class Meta: model = ServiceComponent fields = {"is_overall": ["exact"]} - def filter_ancestor(self, queryset, name, value): - """Match the stored chain, which the GIN index answers. - - `ancestor_ids` holds every step above a row. A containment test - on it reaches any depth in one comparison. - """ - return queryset.filter(ancestor_ids__contains=[value]) - def filter_is_tracked(self, queryset, name, value): """Answer it here, because nobody signed out tracks anything. diff --git a/api/catalog/migrations/0008_component_ancestor_table.py b/api/catalog/migrations/0008_component_ancestor_table.py new file mode 100644 index 0000000..45c3285 --- /dev/null +++ b/api/catalog/migrations/0008_component_ancestor_table.py @@ -0,0 +1,123 @@ +# Generated by Django 5.2.17 on 2026-09-04 06:08 + +import uuid + +import django.db.models.deletion +from django.db import migrations, models + + +def fill_from_ancestor_ids(apps, schema_editor): + """Copy the array into the table, before the column goes. + + Migration 0004 backfilled the array with a recursive walk of + `parent`. Reading it again here makes the two agree by + construction. Recomputing would be a second answer to the same + question. + + The array can name a component somebody deleted. A foreign key + cannot, so such a step is dropped. That dangling reference is the + defect this table removes. + + Python, not SQL, because the key defaults to `uuid.uuid7`. Postgres + 17 has no function for version 7. + """ + ServiceComponent = apps.get_model("catalog", "ServiceComponent") + ComponentAncestor = apps.get_model("catalog", "ComponentAncestor") + live = set(ServiceComponent.objects.values_list("id", flat=True)) + links = [ + ComponentAncestor( + ancestor_id=up, + descendant_id=row_id, + # The array reads root first, so the last step is the parent. + depth=len(ancestors) - step, + ) + for row_id, ancestors in ServiceComponent.objects.values_list( + "id", "ancestor_ids" + ) + for step, up in enumerate(ancestors) + if up in live + ] + ComponentAncestor.objects.bulk_create(links, batch_size=1000) + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0007_drop_service_columns"), + ] + + operations = [ + migrations.CreateModel( + name="ComponentAncestor", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid7, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("depth", models.PositiveIntegerField()), + ], + options={ + "ordering": ["-depth"], + }, + ), + migrations.AddField( + model_name="componentancestor", + name="ancestor", + field=models.ForeignKey( + db_index=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="descendant_links", + to="catalog.servicecomponent", + ), + ), + migrations.AddField( + model_name="componentancestor", + name="descendant", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="ancestor_links", + to="catalog.servicecomponent", + ), + ), + migrations.AddConstraint( + model_name="componentancestor", + constraint=models.UniqueConstraint( + fields=("ancestor", "descendant"), name="one_link_per_component_pair" + ), + ), + migrations.AddConstraint( + model_name="componentancestor", + constraint=models.CheckConstraint( + condition=models.Q(("ancestor", models.F("descendant")), _negated=True), + name="ancestry_excludes_self", + ), + ), + migrations.AddConstraint( + model_name="componentancestor", + constraint=models.CheckConstraint( + condition=models.Q(("depth__gte", 1)), + name="ancestry_depth_starts_at_one", + ), + ), + # The read comes before the drop. The column is the only record + # of the tree that survives a service nobody polls. + migrations.RunPython( + fill_from_ancestor_ids, reverse_code=migrations.RunPython.noop + ), + migrations.RemoveIndex( + model_name="servicecomponent", + name="components_by_ancestor", + ), + migrations.RemoveField( + model_name="historicalservicecomponent", + name="ancestor_ids", + ), + migrations.RemoveField( + model_name="servicecomponent", + name="ancestor_ids", + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index ace0b94..7936d4c 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -1,13 +1,12 @@ +import uuid from contextlib import contextmanager from contextvars import ContextVar from urllib.parse import urlparse, urlunparse -from django.contrib.postgres.fields import ArrayField from django.contrib.postgres.indexes import GinIndex from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVectorField from django.core.exceptions import ValidationError from django.db import models -from django.db.models.functions import Coalesce from django.utils import timezone from django.utils.text import slugify from simple_history.models import HistoricalRecords @@ -235,28 +234,11 @@ def __str__(self): def _descendant_count(): """How many components sit under each row, in one query for the page. - `related_count` cannot express this. It compares a column to the - outer key. `ancestor_ids` holds an array, so the outer key goes in - as a one element array of its own. - - The constant in `values` is what makes this one aggregate. Without - it Django counts per row and returns one row a component, which a - subquery in a SELECT list refuses. + Archived rows are left out. A caller is not served one anywhere + else, so a count that included them would name components no list + returns. """ - key = models.Func( - models.OuterRef("pk"), - template="ARRAY[%(expressions)s]", - output_field=ArrayField(models.UUIDField()), - ) - counted = ( - ServiceComponent.objects.visible() - .filter(ancestor_ids__contains=key) - .order_by() - .values(one_group=models.Value(1)) - .annotate(total=models.Count("pk")) - .values("total") - ) - return Coalesce(models.Subquery(counted, output_field=models.IntegerField()), 0) + return related_count(ComponentAncestor.objects.to_visible(), "ancestor") _rebuilds_held = ContextVar("catalog_rebuilds_held", default=False) @@ -371,13 +353,6 @@ class ServiceComponent(BaseModel): on_delete=models.SET_NULL, related_name="children", ) - # The chain above this one, root first, written by reconcile. A - # component's Components tab lists every descendant, and a self-FK - # lookup returns one level. Walking the chain per row cannot use an - # index either. - ancestor_ids = ArrayField( - models.UUIDField(), default=list, blank=True, editable=False - ) # The whole path, weighted, written by reconcile. Searching a # service's name must also reach its components. Walking `parent` # at query time cannot use an index. @@ -423,7 +398,6 @@ class Meta(BaseModel.Meta): ), ] indexes = [ - GinIndex(fields=["ancestor_ids"], name="components_by_ancestor"), GinIndex(fields=["search_document"], name="components_by_search"), ] @@ -451,11 +425,7 @@ def descendant_count(self): return 0 prepared = getattr(self, "_descendant_count", None) if prepared is None: - return ( - ServiceComponent.objects.visible() - .filter(ancestor_ids__contains=[self.pk]) - .count() - ) + return self.descendant_links.to_visible().count() return prepared def is_tracked_by(self, user): @@ -467,8 +437,8 @@ def is_tracked_by(self, user): return self.boards.filter(owner=user).exists() return prepared - # `ancestor_ids` and `search_document` are built from these. A save - # touching none of them leaves both true. + # The ancestry links and `search_document` are built from these. A + # save touching none of them leaves both true. REBUILD_TRIGGERS = frozenset( {"name", "parent", "parent_id", "service", "service_id"} ) @@ -531,8 +501,8 @@ def _services_to_rebuild(self, update_fields): return () if was[2] == self.service_id: return (self.service,) - # A move between services leaves the old one still naming this - # row in its `ancestor_ids` and in its search documents. + # A move between services leaves the old one behind. Its + # ancestry links and its search documents still name this row. return (self.service, Service.objects.get(pk=was[2])) def clean(self): @@ -575,3 +545,85 @@ def path(self): def __str__(self): return self.path + + +class ComponentAncestorQuerySet(models.QuerySet): + def to_visible(self): + """The links a caller is served, anywhere. + + `ServiceComponentQuerySet.visible` is the one definition of + which components a reader sees. Naming the archive flag here + would give a descendant count a second answer to that. + """ + return self.filter(descendant__in=ServiceComponent.objects.visible()) + + +class ComponentAncestor(models.Model): + """One component sits above another, at a known distance. + + Read the two names off the row, not off the component you hold. + They swap sides depending on where you stand. So a component's own + ancestors are `component.ancestor_links`, and everything under it is + `component.descendant_links`. + + `depth` counts the steps down from `ancestor` to `descendant`. A + parent is 1 and a grandparent is 2. A breadcrumb reads root first + with `order_by("-depth")`, and `depth=1` is the direct children. + + A component is not its own ancestor. Every reader counts other + components, so a depth 0 row would add one to every answer. + + This is not a `BaseModel`. Reconcile derives every row from + `parent`, so an author and an edit time record nothing anybody + reads. The key still follows the house rule. + + The pair replaces an array of ancestor ids. The array held no + reference. Deleting a component left its id in every descendant, + until the next poll of that service. An untracked service is never + polled again. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid7, editable=False) + # The delete rule is the reason this table exists. A component that + # goes takes every claim about where it sat with it. + # + # `db_index=False` because the unique constraint already indexes + # this column first. That index answers the descendant direction: + # `?ancestor=`, and the count on every component row. + ancestor = models.ForeignKey( + ServiceComponent, + on_delete=models.CASCADE, + related_name="descendant_links", + db_index=False, + ) + # The foreign key index answers the ancestor direction: one + # component's own chain, which a breadcrumb reads. + descendant = models.ForeignKey( + ServiceComponent, + on_delete=models.CASCADE, + related_name="ancestor_links", + ) + depth = models.PositiveIntegerField() + objects = ComponentAncestorQuerySet.as_manager() + + class Meta: + # Root first, which is how a breadcrumb reads. + ordering = ["-depth"] + constraints = [ + models.UniqueConstraint( + fields=["ancestor", "descendant"], name="one_link_per_component_pair" + ), + # Both hold "no component is its own ancestor", which every + # count depends on. One forbids the row, one forbids the + # distance that would describe it. + models.CheckConstraint( + condition=~models.Q(ancestor=models.F("descendant")), + name="ancestry_excludes_self", + ), + models.CheckConstraint( + condition=models.Q(depth__gte=1), name="ancestry_depth_starts_at_one" + ), + ] + + def __str__(self): + return f"{self.ancestor} > {self.descendant} ({self.depth})" diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index 3c90877..3b93c7b 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -4,7 +4,7 @@ from django.db.models import Value from django.utils import timezone -from catalog.models import ServiceComponent, held_rebuilds +from catalog.models import ComponentAncestor, ServiceComponent, held_rebuilds from polling.system_events import claim, reconcile_system_events from status.models import ComponentStatus, EventUpdate, ServiceEvent @@ -92,16 +92,12 @@ def _upsert_components(service, components, author): return rows -def rebuild_ancestry(service): - """Write every component's chain of ancestors, root first. +def _chains(rows): + """Each row's ancestors, root first, keyed by id. - A rename does not touch this. A reparent moves a whole subtree, so - the pass rewrites the service rather than one row. A service holds - hundreds of components, not millions. + The walk reads the rows in hand, not the database. A deep tree costs + no extra query, and both rebuilds ask the same question once. """ - rows = list(ServiceComponent.objects.filter(service=service)) - # The walk reads this map, not the database. A deep tree costs no - # extra query. parents = {row.id: row.parent_id for row in rows} def chain(node): @@ -119,15 +115,42 @@ def chain(node): up = parents.get(up) return list(reversed(walked)) - # Only the rows that moved. Most polls change no parent, and this - # runs on every poll of every service. - changed = [] - for row in rows: - computed = chain(row.id) - if row.ancestor_ids != computed: - row.ancestor_ids = computed - changed.append(row) - ServiceComponent.objects.bulk_update(changed, ["ancestor_ids"]) + return {row.id: chain(row.id) for row in rows} + + +def rebuild_ancestry(service): + """Write one link per pair of components, one above the other. + + A rename does not touch this. A reparent moves a whole subtree, so + the pass rewrites the service rather than one row. A service holds + hundreds of components, not millions. + + Only the rows that moved are written. A delete and a re-insert + would rewrite every link of the service on every poll. Almost no + poll moves a component. + """ + rows = list(ServiceComponent.objects.filter(service=service)) + # Root first, so the last step is the parent and depth counts down. + wanted = { + (up, row_id, len(chain) - step) + for row_id, chain in _chains(rows).items() + for step, up in enumerate(chain) + } + held = { + (link.ancestor_id, link.descendant_id, link.depth): link.pk + for link in ComponentAncestor.objects.filter(descendant__service=service) + } + stale = [pk for link, pk in held.items() if link not in wanted] + if stale: + # Before the insert. A pair whose depth changed holds the unique + # constraint until its old row goes. + ComponentAncestor.objects.filter(pk__in=stale).delete() + ComponentAncestor.objects.bulk_create( + [ + ComponentAncestor(ancestor_id=up, descendant_id=row_id, depth=depth) + for up, row_id, depth in wanted - held.keys() + ] + ) def rebuild_search(service): @@ -138,15 +161,15 @@ def rebuild_search(service): service. Ancestor names are joined here, not in SQL. Postgres cannot build a - vector from an array of foreign keys. It must run after - `rebuild_ancestry`, which supplies `ancestor_ids`. + vector from a set of foreign keys. """ rows = list( ServiceComponent.objects.filter(service=service).select_related("service") ) names = {row.id: row.name for row in rows} + chains = _chains(rows) for row in rows: - ancestors = " ".join(names.get(a, "") for a in row.ancestor_ids) + ancestors = " ".join(names[up] for up in chains[row.id]) row.search_document = ( SearchVector(Value(row.name), weight="A") + SearchVector(Value(ancestors), weight="B") diff --git a/api/tests/factories.py b/api/tests/factories.py index 13d494c..6ac6c51 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -88,6 +88,19 @@ def track(component, user=None): return DashboardItem.objects.create(dashboard=board, component=component) +def ancestry(component): + """The ids above it, root first, read from the closure table. + + `depth` counts the steps down to the component, so the largest is + the root. A breadcrumb reads the same order. + """ + return list( + component.ancestor_links.order_by("-depth").values_list( + "ancestor_id", flat=True + ) + ) + + def watchers(component): """How many people track it, counted the way the app counts.""" from catalog.queries import COMPONENT_WATCHER_COUNT diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 835810a..6a9898f 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -920,7 +920,7 @@ def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): by `?q=`. An untracked service is never polled to repair it. """ from catalog.models import ServiceComponent - from tests.factories import ComponentFactory, ServiceFactory + from tests.factories import ComponentFactory, ServiceFactory, ancestry service = ServiceFactory(name="Twilio") parent = ComponentFactory(service=service, name="Programmable Messaging") @@ -930,5 +930,5 @@ def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): assert staff_client.post(url, data).status_code == 302 child.refresh_from_db() - assert child.ancestor_ids == [parent.pk] + assert ancestry(child) == [parent.pk] assert list(ServiceComponent.objects.search("programmable sms")) == [child] diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index e7dc48d..31df33c 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -1,13 +1,15 @@ import pytest +from django.db.models import Q from django.db.utils import IntegrityError -from catalog.models import Service, ServiceComponent, StatusPage +from catalog.models import ComponentAncestor, Service, ServiceComponent, StatusPage from polling.reconcile import rebuild_ancestry, rebuild_search from tests.factories import ( ComponentFactory, PollerFactory, ServiceFactory, UserFactory, + ancestry, track, watchers, ) @@ -171,13 +173,13 @@ def test_a_reparent_in_plain_python_rebuilds_the_derived_columns(db): child.save() child.refresh_from_db() - assert child.ancestor_ids == [parent.pk] + assert ancestry(child) == [parent.pk] assert list(ServiceComponent.objects.search("programmable sms")) == [child] def test_a_move_between_services_rebuilds_the_service_left_behind(db): # Only the new service was rebuilt. The old one kept rows naming - # this component in `ancestor_ids` and in their search documents. + # this component in their ancestry and in their search documents. old = ServiceFactory(name="Twilio") new = ServiceFactory(name="Vonage") top = ComponentFactory(service=old, name="Programmable Messaging") @@ -191,7 +193,7 @@ def test_a_move_between_services_rebuilds_the_service_left_behind(db): top.save() leaf.refresh_from_db() - assert leaf.ancestor_ids == [] + assert ancestry(leaf) == [] assert list(ServiceComponent.objects.search("programmable sms")) == [] @@ -212,8 +214,8 @@ def test_a_save_that_moves_nothing_rebuilds_nothing(db, monkeypatch): assert calls == [] -def test_ancestor_ids_are_stored_top_down(db): - # A descendant query reads this array. Walking `parent` per row +def test_ancestry_records_one_link_a_step_with_the_distance(db): + # A descendant query joins this table. Walking `parent` per row # cannot use an index, and `for_display` only selects two levels up. service = ServiceFactory() top = ComponentFactory(service=service) @@ -221,11 +223,42 @@ def test_ancestor_ids_are_stored_top_down(db): leaf = ComponentFactory(service=service, parent=middle) rebuild_ancestry(service) - leaf.refresh_from_db() - assert leaf.ancestor_ids == [top.id, middle.id] + # Root first, which is the order a breadcrumb reads. + assert ancestry(leaf) == [top.id, middle.id] + # The distance, so one query can ask for the direct children alone. + # A parent is 1. Counting from the root instead would make `depth=1` + # name the top of the tree. + assert {link.ancestor_id: link.depth for link in leaf.ancestor_links.all()} == { + top.id: 2, + middle.id: 1, + } # Every descendant, not the direct children. `top` has two below it. - assert ServiceComponent.objects.filter(ancestor_ids__contains=[top.id]).count() == 2 + assert ServiceComponent.objects.filter(ancestor_links__ancestor=top).count() == 2 + + +def test_deleting_a_component_takes_its_ancestry_with_it(db): + # An array of ids held no reference. Deleting a component left its + # id in every descendant, and only a poll of that service cleared + # it. An untracked service is never polled again. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + + # Read the key first. A delete clears it on the instance, so a + # filter written afterwards matches nothing and proves nothing. + gone = middle.pk + middle.delete() + + assert not ComponentAncestor.objects.filter( + Q(ancestor=gone) | Q(descendant=gone) + ).exists() + # `parent` is SET_NULL, so the leaf is now a root. The link to its + # grandparent outlives the delete, and the next rebuild drops it. + rebuild_ancestry(service) + assert ancestry(leaf) == [] def test_for_display_counts_descendants_without_a_query_a_row( diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index b0e87ac..e4aeaee 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -16,7 +16,7 @@ StatusSource, ) from status.models import ComponentStatus, ServiceEvent -from tests.factories import ComponentFactory, ServiceFactory +from tests.factories import ComponentFactory, ServiceFactory, ancestry def _component(external_id="a", name="SMS", severity=Severity.OPERATIONAL, **kw): @@ -310,8 +310,8 @@ def test_a_poll_writes_the_ancestor_chain(): top = ServiceComponent.objects.get(external_id="a") child = ServiceComponent.objects.get(external_id="b") - assert top.ancestor_ids == [] - assert child.ancestor_ids == [top.id] + assert ancestry(top) == [] + assert ancestry(child) == [top.id] @pytest.mark.django_db @@ -358,7 +358,7 @@ def test_a_reparent_rewrites_the_subtree(): ] apply_fetch(service, moved, [], StatusSource.PROVIDER) - assert ServiceComponent.objects.get(external_id="c").ancestor_ids == [ + assert ancestry(ServiceComponent.objects.get(external_id="c")) == [ ServiceComponent.objects.get(external_id="b").id ] @@ -376,10 +376,8 @@ def test_a_loop_in_the_parent_column_does_not_hang_the_pass(): rebuild_ancestry(service) - first.refresh_from_db() - second.refresh_from_db() - assert first.ancestor_ids == [second.id] - assert second.ancestor_ids == [first.id] + assert ancestry(first) == [second.id] + assert ancestry(second) == [first.id] @pytest.mark.django_db @@ -401,7 +399,6 @@ def test_a_reparent_rewrites_every_descendants_path(): rebuild_ancestry(service) rebuild_search(service) - leaf.refresh_from_db() - assert leaf.ancestor_ids == [new.id] + assert ancestry(leaf) == [new.id] assert list(ServiceComponent.objects.search("legacy queue")) == [] assert list(ServiceComponent.objects.search("platform queue")) == [leaf] From 9eee63eacc73f15ca0be32f3c7c075a6e86fd208 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 11:24:07 -0700 Subject: [PATCH 34/94] docs: the specs describe the ancestry table, not the array The ER diagram named a column the code no longer has, and the client design still argued for it. Section 8 claimed one column paid for three readers, which stopped being true when the readers moved to a table. The argument for the array was an indexed containment test. The table keeps that and adds the referential integrity the array could not hold, which is the reason for the change. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- docs/check_docs.py | 2 +- docs/specs/2026-08-23-statusboard-design.md | 23 ++++++++--- .../2026-09-03-statusboard-client-design.md | 40 +++++++++++++++---- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/check_docs.py b/docs/check_docs.py index 37cc1dd..f6529ef 100755 --- a/docs/check_docs.py +++ b/docs/check_docs.py @@ -62,7 +62,7 @@ "Poller.cooldown_seconds": "own override or /meta/ default", "Service.poller": "OneToOneField", "Component.path": "walk of parent", - "Component.descendant_count": "Count of rows whose ancestor_ids holds this id", + "Component.descendant_count": "Count of ComponentAncestor where ancestor is this id", "Component.is_tracked": "per-user annotation over DashboardItem", "Component.upcoming_maintenance": "soonest unfinished maintenance event across that M2M", "Component.upcoming_maintenance_count": "Count of unfinished maintenance across that M2M", diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index fbf0c9e..ec01cdd 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -76,8 +76,9 @@ what makes sharing a change to `dashboards` alone. ## 3. Data model Every model extends `common.BaseModel`: UUID primary key, `created_at`, `updated_at`, -`created_by`, `updated_by`, `ordering = ["-created_at"]`. `auth.Permission` and `ContentType` -keep integer keys — that is Django, not a choice. +`created_by`, `updated_by`, `ordering = ["-created_at"]`. `ComponentAncestor` is the one +exception: reconcile derives every row from `parent`, so an author records nothing. +`auth.Permission` and `ContentType` keep integer keys — that is Django, not a choice. ### authentication @@ -154,13 +155,18 @@ erDiagram string external_id "unique with service" string name uuid parent_id FK "self, null at top level" - uuid[] ancestor_ids "root first, written by reconcile" tsvector search_document "weighted path, written by reconcile" int status_page_order bool is_overall bool is_featured "first key of the suggested sort" datetime archived_at "set when it stops being published" } + ComponentAncestor { + uuid id PK + uuid ancestor_id FK "the component above" + uuid descendant_id FK "the component below, unique with ancestor" + int depth "steps down to the descendant, a parent is 1" + } ComponentStatus { uuid id PK uuid component_id FK @@ -212,6 +218,8 @@ erDiagram Service ||--|| Poller : "read by" Service ||--o{ ServiceComponent : publishes ServiceComponent ||--o| ServiceComponent : "parent of" + ServiceComponent ||--o{ ComponentAncestor : "sits above" + ServiceComponent ||--o{ ComponentAncestor : "sits below" ServiceComponent ||--o{ ComponentStatus : "severity over time" Poller ||--o{ PollRun : "attempts" Service ||--o{ ServiceEvent : "published on" @@ -219,8 +227,9 @@ erDiagram ServiceEvent }o--o{ ServiceComponent : affects ``` -Every model also carries `common.BaseModel`: `created_at`, `updated_at`, `created_by`, -`updated_by`. They are omitted above so the diagram shows what distinguishes each table. +Every model above but `ComponentAncestor` also carries `common.BaseModel`: `created_at`, +`updated_at`, `created_by`, `updated_by`. They are omitted so the diagram shows what +distinguishes each table. Reading it: @@ -228,6 +237,10 @@ Reading it: these, so the arrow from `DashboardItem` has one destination. - **`StatusPage` is separate from `Service`** because it is the *source*, and it is where the unique URL and the poller's state live. +- **`ComponentAncestor` is the tree, flattened**, one row per pair with one component above the + other. `parent` alone answers one level; this answers any depth in a single indexed join, which + is what `?ancestor=` and every descendant count ask for. Both columns are real foreign keys, so + deleting a component takes every claim about where it sat with it. - **`ComponentStatus` is append-only**, one row per severity change, with a partial unique index marking the current one. Current state and history are the same table. diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md index e90270c..ed766e8 100644 --- a/docs/specs/2026-09-03-statusboard-client-design.md +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -248,7 +248,7 @@ that does not exist until you type. ## 6. Model changes -### New model +### New models `catalog.ServiceRequest`, for "Send this URL to us". @@ -274,12 +274,36 @@ once a URL was triaged. for copying `PollRun.error`, which can disagree with its source. Here there is no source. The counter is the record. +`catalog.ComponentAncestor`, the component tree flattened. + +| Field | Note | +| --- | --- | +| `ancestor` | FK to `ServiceComponent`, the one above | +| `descendant` | FK to `ServiceComponent`, the one below. Unique with `ancestor`. | +| `depth` | the steps down from `ancestor` to `descendant`. A parent is 1. | + +One row per pair, written by reconcile. `parent` answers one level. This answers any depth +in one indexed join, which is what `?ancestor=` and every `descendant_count` ask for. A +breadcrumb reads the same rows the other way round, root first by `-depth`. + +**Both columns are foreign keys, and that is the point.** An array of ancestor ids buys the +same indexed test and holds no reference. `parent` is `SET_NULL`, so nothing cleans an array +when a component goes, and every descendant keeps naming a row that is gone. Only a poll of +that service repairs it, and an untracked service is never polled again. A foreign key +cascades in the same statement as the delete. + +It is not a `BaseModel`. Reconcile derives every row from `parent`, so an author and an edit +time record nothing anybody reads. The UUIDv7 key stays. + +No row says a component is its own ancestor. Every reader counts other components, so a +depth-zero row would add one to every answer. Two check constraints hold that: one forbids +the row, one forbids the distance that would describe it. + ### New columns | Column | Type | Written by | | --- | --- | --- | | `ServiceComponent.is_featured` | bool | admin | -| `ServiceComponent.ancestor_ids` | `ArrayField(UUIDField)`, GIN index | reconcile | | `ServiceComponent.search_document` | `SearchVectorField`, GIN index | reconcile | | `ServiceEvent.detected_by` | `provider` or `system` | set when the event opens, never changed | | `EventUpdate.source` | `provider` or `system` | the adapter, or reconcile | @@ -411,18 +435,20 @@ query time cannot use an index. | Weight | Source | | --- | --- | | `A` | the component's own name | -| `B` | its ancestors' names, read from `ancestor_ids` | +| `B` | its ancestors' names, walked from `parent` in that pass | | `C` | its service's name | So `twilio` ranks the Twilio rollup first, then top-level components, then leaves. The ranking solves most of the flooding a popular service would cause. `q` becomes a `SearchQuery` in websearch mode, so several words are an AND, ordered by `SearchRank`. -**Renaming or reparenting rewrites a whole subtree.** `ancestor_ids` and `search_document` -both carry ancestry, so both change. Providers do move components, so that rewrite belongs -inside the reconcile pass. This is the one place the design fans out. +**Renaming or reparenting rewrites a whole subtree.** `ComponentAncestor` and +`search_document` both carry ancestry, so both change. Providers do move components, so that +rewrite belongs inside the reconcile pass. This is the one place the design fans out. -One column pays for three readers: the descendant query, the breadcrumb, and this weight. +Both read one walk of `parent`, over the rows the pass already holds. A deep tree costs no +extra query. The ancestry rebuild then writes only the links that moved, because almost no +poll moves a component. The service `search_fields` go with the service list. From 7408696bb8670772c2a7624106c08d84d832424f Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 11:35:34 -0700 Subject: [PATCH 35/94] fix: deleting a component rebuilds its service A mid-tree delete left the grandparent link behind. `parent` is SET_NULL, so the child became a root while the closure table still placed it under a grandparent. `?ancestor=` then named a tree nobody sits in, and every descendant count above it read one too many. The search path went stale the same way. A deleted parent's name still found the component it no longer held. Nothing called `rebuild_ancestry` on the way out. `SET_NULL` is a queryset update, so the `save` hook never fires. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/models.py | 21 +++++++++++++++++++++ api/tests/test_catalog_models.py | 23 ++++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/api/catalog/models.py b/api/catalog/models.py index 7936d4c..b4d385d 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -472,6 +472,27 @@ def save(self, *args, **kwargs): rebuild_ancestry(service) rebuild_search(service) + def delete(self, *args, **kwargs): + """Rebuild the service, because a delete moves the rows below. + + The cascade drops this row's own links. `parent` is SET_NULL, + so a child of this row becomes a root. The link from this row's + parent down to that child survives. It names a tree nobody + sits in. + + A queryset delete does not come through here. Nothing removes a + component that way. A poll archives instead. + + Read the service before the write. Django clears the key. + """ + service = self.service + result = super().delete(*args, **kwargs) + from polling.reconcile import rebuild_ancestry, rebuild_search + + rebuild_ancestry(service) + rebuild_search(service) + return result + def _services_to_rebuild(self, update_fields): """Whose derived columns this save is about to invalidate. diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 31df33c..34d4874 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -255,12 +255,29 @@ def test_deleting_a_component_takes_its_ancestry_with_it(db): assert not ComponentAncestor.objects.filter( Q(ancestor=gone) | Q(descendant=gone) ).exists() - # `parent` is SET_NULL, so the leaf is now a root. The link to its - # grandparent outlives the delete, and the next rebuild drops it. - rebuild_ancestry(service) + # `parent` is SET_NULL, so the leaf is now a root. The cascade + # clears the deleted row's own links only. Without the rebuild the + # leaf keeps a grandparent it no longer sits under. assert ancestry(leaf) == [] +def test_deleting_a_component_takes_its_name_out_of_the_search_path(db): + # `search_document` holds each ancestor's name. Left stale, a + # deleted parent's name still finds a component under nothing. + service = ServiceFactory(name="Twilio") + top = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=top) + rebuild_ancestry(service) + rebuild_search(service) + + top.delete() + + assert list(ServiceComponent.objects.search("messaging")) == [] + # The leaf still answers its own name. A rebuild that dropped the + # whole vector would pass the line above and serve nothing. + assert list(ServiceComponent.objects.search("sms")) == [leaf] + + def test_for_display_counts_descendants_without_a_query_a_row( db, django_assert_num_queries ): From 65d5d0052470d50494af10753b79544d279da04e Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 11:41:42 -0700 Subject: [PATCH 36/94] refactor: the display annotations carry their plain names The fallback methods let a serving path skip `for_display` and pay a query a row in silence. Two did: the board track write and the catalog import both serialized a row straight from the write. Each now reads the row back through `for_display`. With the fallbacks gone the underscore has nothing to dodge, so the annotations take the names the serializer reads. `is_tracked` was never private either. `catalog.filters` reads it from another module, and the contract documents it as a query parameter. `_live_events` keeps its underscore. Its reader is `live_events`, which is the one accessor that shares a name with what it reads. A path that forgets `for_display` now raises `AttributeError`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/filters.py | 4 +- api/catalog/models.py | 69 +++++++++++--------------------- api/catalog/serializers.py | 8 ++-- api/catalog/views.py | 6 ++- api/dashboards/views.py | 10 ++++- api/tests/test_board_api.py | 14 +++++++ api/tests/test_catalog_api.py | 8 ++-- api/tests/test_catalog_models.py | 33 +++++++-------- 8 files changed, 76 insertions(+), 76 deletions(-) diff --git a/api/catalog/filters.py b/api/catalog/filters.py index 5d68d4c..5e35c6a 100644 --- a/api/catalog/filters.py +++ b/api/catalog/filters.py @@ -29,10 +29,10 @@ def filter_is_tracked(self, queryset, name, value): """Answer it here, because nobody signed out tracks anything. Both directions are true for that reader, so neither is an - error. `for_display` annotates `_is_tracked` as NULL then, and + error. `for_display` annotates `is_tracked` as NULL then, and SQL reads `NOT NULL` as NULL rather than true. Comparing against the annotation would drop every row from Untracked. """ if self.request is None or self.request.user.is_anonymous: return queryset.none() if value else queryset - return queryset.filter(_is_tracked=value) + return queryset.filter(is_tracked=value) diff --git a/api/catalog/models.py b/api/catalog/models.py index b4d385d..343959b 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -35,11 +35,11 @@ def for_display(self, user=None): return ( self.select_related("status_page", "poller") .annotate( - _component_count=related_count( + component_count=related_count( ServiceComponent.objects.visible().filter(is_overall=False), "service", ), - _tracked_component_count=tracked, + tracked_component_count=tracked, ) .prefetch_related( models.Prefetch( @@ -47,7 +47,7 @@ def for_display(self, user=None): queryset=ServiceComponent.objects.filter( is_overall=True ).for_display(user), - to_attr="_overall_components", + to_attr="overall_components", ) ) ) @@ -79,36 +79,17 @@ def save(self, *args, **kwargs): def overall_component(self): """The rollup component, which is the service's own status. - This and the two below read what `for_display` prepared. A row - fetched without it asks instead. The underscore names are that - queryset's, so a rename cannot leave a reader behind. + `for_display` prefetches it. A row fetched without that asks + instead, because a single service nests one. + + The name is singular against the plural prefetch, so the two + never collide. """ - prepared = getattr(self, "_overall_components", None) + prepared = getattr(self, "overall_components", None) if prepared is None: return self.components.filter(is_overall=True).first() return prepared[0] if prepared else None - def component_count(self): - """The parts. The rollup is excluded, because it is the service.""" - prepared = getattr(self, "_component_count", None) - if prepared is not None: - return prepared - return self.components.visible().filter(is_overall=False).count() - - def tracked_component_count(self, user): - """How many parts this user has on a board.""" - if user is None or not user.is_authenticated: - return 0 - prepared = getattr(self, "_tracked_component_count", None) - if prepared is not None: - return prepared - return ( - ServiceComponent.objects.visible() - .filter(service=self, boards__owner=user) - .distinct() - .count() - ) - def ensure_poller(self): """Every service is polled, so every service has one. @@ -298,8 +279,9 @@ def for_display(self, user=None): It answered seven fields with a query each, per row. A page of fifty cost three hundred and fifty six. - A row that skipped this still serializes, because a single - service nests one. So the fast path has to be asked for. + Every path that serializes a component calls this. A path that + forgets raises `AttributeError` on the first count, rather than + running one query a row in silence. """ from django.contrib.auth.models import AnonymousUser @@ -318,8 +300,8 @@ def for_display(self, user=None): return ( self.select_related("service", "parent", "parent__parent") .annotate( - _descendant_count=_descendant_count(), - _is_tracked=tracked, + descendant_count=_descendant_count(), + is_tracked=tracked, ) .prefetch_related( models.Prefetch( @@ -329,11 +311,14 @@ def for_display(self, user=None): queryset=ComponentStatus.objects.filter( ended_at__isnull=True ).select_related("component__service__poller"), - to_attr="_open_statuses", + to_attr="open_statuses", ), models.Prefetch( "events", queryset=ServiceEvent.objects.live(), + # The one name that keeps its underscore. Its reader + # is `live_events`, so the plain name would shadow + # the method that reads it. to_attr="_live_events", ), ) @@ -404,8 +389,9 @@ class Meta(BaseModel.Meta): def live_events(self, kind): """The live events of one kind. - This and the three below read what `for_display` prepared. A row - fetched without it asks instead. + This and the two below read what `for_display` prepared. Each + picks one row, or one kind, out of a prefetch. So the method + and the attribute cannot share a name. """ prepared = getattr(self, "_live_events", None) if prepared is None: @@ -414,25 +400,16 @@ def live_events(self, kind): def open_status(self): """The status span still running, if there is one.""" - prepared = getattr(self, "_open_statuses", None) + prepared = getattr(self, "open_statuses", None) if prepared is None: return self.statuses.filter(ended_at__isnull=True).first() return prepared[0] if prepared else None - def descendant_count(self): - """How many components sit anywhere under this one.""" - if self.is_overall: - return 0 - prepared = getattr(self, "_descendant_count", None) - if prepared is None: - return self.descendant_links.to_visible().count() - return prepared - def is_tracked_by(self, user): """Whether this user has it on a board. Null if nobody asked.""" if user is None or not user.is_authenticated: return None - prepared = getattr(self, "_is_tracked", None) + prepared = getattr(self, "is_tracked", None) if prepared is None: return self.boards.filter(owner=user).exists() return prepared diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index 30a0289..40e4712 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -114,7 +114,7 @@ def get_path(self, row): @extend_schema_field(serializers.IntegerField()) def get_descendant_count(self, row): - return row.descendant_count() + return row.descendant_count @extend_schema_field(EventRefSerializer(allow_null=True)) def get_upcoming_maintenance(self, row): @@ -157,7 +157,7 @@ def get_is_tracked(self, row): return row.is_tracked_by(self.viewer) -class ServiceSerializer(ViewerMixin, FieldsMixin, serializers.ModelSerializer): +class ServiceSerializer(FieldsMixin, serializers.ModelSerializer): status_page = StatusPageSerializer(read_only=True) poller = PollerSerializer(read_only=True) overall_component = serializers.SerializerMethodField() @@ -196,11 +196,11 @@ def get_overall_component(self, service): @extend_schema_field(serializers.IntegerField()) def get_component_count(self, service): - return service.component_count() + return service.component_count @extend_schema_field(serializers.IntegerField()) def get_tracked_component_count(self, service): - return service.tracked_component_count(self.viewer) + return service.tracked_component_count class ImportRequestSerializer(serializers.Serializer): diff --git a/api/catalog/views.py b/api/catalog/views.py index ea84557..4b72aca 100644 --- a/api/catalog/views.py +++ b/api/catalog/views.py @@ -74,8 +74,12 @@ def post(self, request): raise NoStatusPageFound(str(error)) from error except requests.RequestException as error: raise ProviderUnreachable(f"{url} could not be read: {error}") from error + # Read again, after the import. `ServiceSerializer` reads the + # counts `for_display` annotates, and the import returns a + # plain row. + row = Service.objects.for_display(request.user).get(pk=service.pk) return Response( - ServiceSerializer(service, context={"request": request}).data, + ServiceSerializer(row, context={"request": request}).data, status=http.HTTP_201_CREATED if created else http.HTTP_200_OK, ) diff --git a/api/dashboards/views.py b/api/dashboards/views.py index f2f32db..5520856 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -78,8 +78,16 @@ def create(self, request, *args, **kwargs): component=component, defaults={"created_by": request.user, "updated_by": request.user}, ) + # Read again, after the write. `ComponentSerializer` reads the + # counts `for_display` annotates, and `is_tracked` is false + # until the item exists. + row = ( + ServiceComponent.objects.visible() + .for_display(request.user) + .get(pk=component.pk) + ) return Response( - ComponentSerializer(component, context={"request": request}).data, + ComponentSerializer(row, context={"request": request}).data, status=status.HTTP_201_CREATED if created else status.HTTP_200_OK, ) diff --git a/api/tests/test_board_api.py b/api/tests/test_board_api.py index 756a610..ab8deb4 100644 --- a/api/tests/test_board_api.py +++ b/api/tests/test_board_api.py @@ -153,6 +153,20 @@ def test_tracking_a_component_adds_it_and_bumps_the_watcher_count(client, board) assert watchers(component) == 1 +@pytest.mark.django_db +def test_the_tracked_component_comes_back_prepared(client, board): + # The write returns a component row. Its counts are annotations, so + # a fetch that skips `for_display` cannot fill them. + parent = ComponentFactory(service=ServiceFactory()) + ComponentFactory(service=parent.service, parent=parent) + url = reverse("board-components", kwargs={"uuid": board.id}) + body = client.post(url, {"component_id": str(parent.id)}, format="json").json() + assert body["descendant_count"] == 1 + # Read after the write. The annotation is false until the item is + # there, so the order of the two matters. + assert body["is_tracked"] is True + + @pytest.mark.django_db def test_tracking_the_same_component_twice_is_not_an_error(client, board): component = ComponentFactory() diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index bdc6253..8e5f0dd 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -309,6 +309,7 @@ def test_a_component_with_no_reading_sorts_last_not_first(): def test_a_path_carries_enough_to_link_to_each_step(): # A joined string named the ancestors and gave a client nothing to # click. Each step is a row with its own id. + from catalog.models import ServiceComponent from catalog.serializers import ComponentSerializer service = ServiceFactory(name="Twilio") @@ -316,16 +317,17 @@ def test_a_path_carries_enough_to_link_to_each_step(): parent = ComponentFactory( service=service, name="Programmable Messaging", parent=overall ) - child = ComponentFactory(service=service, name="SMS", parent=parent) + ComponentFactory(service=service, name="SMS", parent=parent) - path = ComponentSerializer(child).data["path"] + prepared = ServiceComponent.objects.for_display() + path = ComponentSerializer(prepared.get(name="SMS")).data["path"] assert [step["name"] for step in path] == ["Twilio", "Programmable Messaging"] assert path[0]["id"] == str(overall.id) assert path[0]["is_overall"] is True # Empty, not null: the overall component is under nothing, and a # client maps over the list without checking first. - assert ComponentSerializer(overall).data["path"] == [] + assert ComponentSerializer(prepared.get(pk=overall.pk)).data["path"] == [] @pytest.mark.django_db diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 34d4874..e79a925 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -291,13 +291,13 @@ def test_for_display_counts_descendants_without_a_query_a_row( rows = list(ServiceComponent.objects.for_display()) with django_assert_num_queries(0): - counts = sorted(row.descendant_count() for row in rows) + counts = sorted(row.descendant_count for row in rows) # Two below the top, one below the middle, none below the leaf. assert counts == [0, 1, 2] -def test_the_prepared_descendant_count_leaves_out_an_archived_row(db): +def test_the_descendant_count_leaves_out_an_archived_row(db): # The badge and the list below it read one number. If this counted # an archived row, the tab would say 2 and then show one component. service = ServiceFactory() @@ -311,22 +311,19 @@ def test_the_prepared_descendant_count_leaves_out_an_archived_row(db): leaf.save(update_fields=["is_archived"]) row = ServiceComponent.objects.for_display().get(pk=top.pk) - assert row.descendant_count() == 1 + assert row.descendant_count == 1 -def test_the_unprepared_descendant_count_leaves_out_an_archived_row(db): - # A row fetched without `for_display` counts for itself. Two ways to - # answer one question is how the two answers drift apart. - service = ServiceFactory() - top = ComponentFactory(service=service) - middle = ComponentFactory(service=service, parent=top) - leaf = ComponentFactory(service=service, parent=middle) - rebuild_ancestry(service) - leaf.is_archived = True - leaf.save(update_fields=["is_archived"]) +def test_a_component_read_without_for_display_has_no_count(db): + # The count is an annotation and nothing falls back to a query. A + # serving path that forgets `for_display` has to fail loudly, not + # ask once a row. + top = ComponentFactory() + ComponentFactory(service=top.service, parent=top) - assert ServiceComponent.objects.get(pk=top.pk).descendant_count() == 1 - assert middle.descendant_count() == 0 + row = ServiceComponent.objects.get(pk=top.pk) + with pytest.raises(AttributeError): + assert row.descendant_count def test_a_services_component_count_leaves_out_an_archived_row(db): @@ -338,8 +335,7 @@ def test_a_services_component_count_leaves_out_an_archived_row(db): gone.is_archived = True gone.save() - assert Service.objects.for_display().get(pk=service.pk).component_count() == 1 - assert Service.objects.get(pk=service.pk).component_count() == 1 + assert Service.objects.for_display().get(pk=service.pk).component_count == 1 def test_a_services_tracked_count_leaves_out_an_archived_row(db): @@ -354,8 +350,7 @@ def test_a_services_tracked_count_leaves_out_an_archived_row(db): gone.save() prepared = Service.objects.for_display(user).get(pk=service.pk) - assert prepared.tracked_component_count(user) == 0 - assert Service.objects.get(pk=service.pk).tracked_component_count(user) == 0 + assert prepared.tracked_component_count == 0 def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): From f2d9f285135c6857c28016f2626b3317f82beca9 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 11:42:55 -0700 Subject: [PATCH 37/94] refactor: the descendant count expression lives with the queries It is a bare expression handed to `annotate()`, which is what `queries.py` holds. Left in `models.py` it split one rule across two files, and a reader had to know both to find an annotation. The circular import is solved the way `is_tracked` above it solves one, with a deferred import inside the function. `ServiceComponentQuerySet.search` stays. It is chainable vocabulary on the queryset, not an expression. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/models.py | 13 ++----------- api/catalog/queries.py | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/api/catalog/models.py b/api/catalog/models.py index 343959b..fcc80cd 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -212,16 +212,6 @@ def __str__(self): return self.url -def _descendant_count(): - """How many components sit under each row, in one query for the page. - - Archived rows are left out. A caller is not served one anywhere - else, so a count that included them would name components no list - returns. - """ - return related_count(ComponentAncestor.objects.to_visible(), "ancestor") - - _rebuilds_held = ContextVar("catalog_rebuilds_held", default=False) @@ -285,6 +275,7 @@ def for_display(self, user=None): """ from django.contrib.auth.models import AnonymousUser + from catalog.queries import descendant_count from status.models import ComponentStatus, ServiceEvent tracked = models.Value(None, output_field=models.BooleanField()) @@ -300,7 +291,7 @@ def for_display(self, user=None): return ( self.select_related("service", "parent", "parent__parent") .annotate( - descendant_count=_descendant_count(), + descendant_count=descendant_count(), is_tracked=tracked, ) .prefetch_related( diff --git a/api/catalog/queries.py b/api/catalog/queries.py index 609b2b9..776f987 100644 --- a/api/catalog/queries.py +++ b/api/catalog/queries.py @@ -1,8 +1,7 @@ -"""What a caller reads about a service, as SQL. +"""What a caller reads about the catalog, as SQL. -A service's own severity is its overall component's. Who watches a -component is counted through the boards it sits on. Both are questions -about the catalog, so they live here. +Each one is a bare expression, handed to `annotate()`. Chainable +vocabulary is a queryset method, and lives on the queryset instead. These lived in `common.ordering`, which made the base layer import the app it describes. @@ -10,6 +9,7 @@ from django.db.models import Count, Exists, OuterRef, Subquery +from common.queries import related_count from status.models import ComponentStatus # A service's own status is the open row of its overall component. That @@ -38,3 +38,15 @@ def is_tracked(field="pk"): from dashboards.models import DashboardItem return Exists(DashboardItem.objects.filter(component__service=OuterRef(field))) + + +def descendant_count(): + """How many components sit under each row, in one query for the page. + + Archived rows are left out. A caller is not served one anywhere + else, so a count that included them would name components no list + returns. + """ + from catalog.models import ComponentAncestor + + return related_count(ComponentAncestor.objects.to_visible(), "ancestor") From 63a9a1aaef8adecd4b41631e391bfb0a16710e19 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 11:44:42 -0700 Subject: [PATCH 38/94] refactor: catalog serves its views from a package `views_components.py` was named around a limitation, not a resource. The app serves three, so one `views.py` had grown too big to read and the split leaked into the filename. One module per resource now. The other apps serve one resource each and keep a single `views.py`. `__init__.py` is empty. A re-export layer would give every view two import paths, and a reader could not tell which module holds it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/urls.py | 6 +- api/catalog/views.py | 129 ------------------ api/catalog/views/__init__.py | 0 .../components.py} | 0 api/catalog/views/imports.py | 56 ++++++++ api/catalog/views/requests.py | 56 ++++++++ api/catalog/views/services.py | 26 ++++ api/tests/test_meta.py | 2 +- api/tests/test_service_requests.py | 2 +- 9 files changed, 144 insertions(+), 133 deletions(-) delete mode 100644 api/catalog/views.py create mode 100644 api/catalog/views/__init__.py rename api/catalog/{views_components.py => views/components.py} (100%) create mode 100644 api/catalog/views/imports.py create mode 100644 api/catalog/views/requests.py create mode 100644 api/catalog/views/services.py diff --git a/api/catalog/urls.py b/api/catalog/urls.py index 44df4a7..641e318 100644 --- a/api/catalog/urls.py +++ b/api/catalog/urls.py @@ -1,7 +1,9 @@ from django.urls import path -from catalog.views import CatalogImportView, ServiceDetailView, ServiceRequestView -from catalog.views_components import ComponentDetailView, ComponentListView +from catalog.views.components import ComponentDetailView, ComponentListView +from catalog.views.imports import CatalogImportView +from catalog.views.requests import ServiceRequestView +from catalog.views.services import ServiceDetailView urlpatterns = [ path("catalog/import/", CatalogImportView.as_view(), name="catalog-import"), diff --git a/api/catalog/views.py b/api/catalog/views.py deleted file mode 100644 index 4b72aca..0000000 --- a/api/catalog/views.py +++ /dev/null @@ -1,129 +0,0 @@ -import requests -from django.db.models import F -from django.utils import timezone -from drf_spectacular.utils import OpenApiResponse, extend_schema -from rest_framework import generics -from rest_framework import status as http -from rest_framework.permissions import AllowAny -from rest_framework.response import Response -from rest_framework.views import APIView - -from api.defaults import Throttle -from catalog.models import Service, ServiceRequest, StatusPage -from catalog.serializers import ( - ImportRequestSerializer, - ServiceRequestSerializer, - ServiceSerializer, -) -from common.errors import NoStatusPageFound, ProviderUnreachable -from common.filters import FieldsBackend -from polling.importer import import_from_url - - -class ServiceDetailView(generics.RetrieveAPIView): - """The service page's header and its About tab. - - There is no list. Discover searches components, and the signed-out - board lists overall components. So nothing asked a service - collection a question. - """ - - permission_classes = [AllowAny] - lookup_field = "slug" - serializer_class = ServiceSerializer - filter_backends = [FieldsBackend] - # get_queryset reads the request, which schema generation has not - # set. This names the model without running it. - queryset = Service.objects.none() - - def get_queryset(self): - return Service.objects.for_display(self.request.user) - - -class CatalogImportView(APIView): - """Add a service by pasting the address of its status page. - - Its own path, because the body is a URL rather than a service. A - later create that takes a service body then does not overload this - one. - """ - - permission_classes = [AllowAny] - # Every call makes the server fetch a URL somebody else chose. - throttle_scope = Throttle.IMPORT - - @extend_schema( - request=ImportRequestSerializer, - responses={ - 200: ServiceSerializer, - 201: ServiceSerializer, - 400: OpenApiResponse(description="Missing or malformed status_page_url."), - }, - ) - def post(self, request): - body = ImportRequestSerializer(data=request.data) - if not body.is_valid(): - return Response(body.errors, status=400) - url = body.validated_data["status_page_url"] - try: - service, created = import_from_url(url) - except ValueError as error: - # `identify` raises this when no adapter could read the page. - # It is the ordinary outcome of pasting the wrong address, - # and it used to reach the caller as a 500. - raise NoStatusPageFound(str(error)) from error - except requests.RequestException as error: - raise ProviderUnreachable(f"{url} could not be read: {error}") from error - # Read again, after the import. `ServiceSerializer` reads the - # counts `for_display` annotates, and the import returns a - # plain row. - row = Service.objects.for_display(request.user).get(pk=service.pk) - return Response( - ServiceSerializer(row, context={"request": request}).data, - status=http.HTTP_201_CREATED if created else http.HTTP_200_OK, - ) - - -class ServiceRequestView(APIView): - """ "Send this URL to us", from the Add-by-URL not-found screen. - - An import stores nothing about an attempt that failed, so this is - where a dead end goes. Telling somebody to hunt for a better link - assumes they have not already tried. - """ - - permission_classes = [AllowAny] - # An anonymous write. Without this one person could inflate the - # demand signal the admin list is ordered by. - # - # Its own scope, not IMPORT's. A failed import and the report that - # follows it share one flow. One bucket would let the import spend - # the report's budget. - throttle_scope = Throttle.SERVICE_REQUEST - - @extend_schema( - request=ServiceRequestSerializer, - responses={ - 202: OpenApiResponse(description="Recorded."), - 400: OpenApiResponse(description="Missing or malformed url."), - }, - ) - def post(self, request): - body = ServiceRequestSerializer(data=request.data) - if not body.is_valid(): - return Response(body.errors, status=400) - url = StatusPage.normalise_url(body.validated_data["url"]) - author = request.user if request.user.is_authenticated else None - row, created = ServiceRequest.objects.get_or_create( - url=url, defaults={"created_by": author, "updated_by": author} - ) - if not created: - # F() rather than a read and a write. Two people asking at - # once would otherwise both write 2. - ServiceRequest.objects.filter(pk=row.pk).update( - request_count=F("request_count") + 1, - last_requested_at=timezone.now(), - ) - # Always 202, and always the same body. Anything else would - # tell a stranger which URLs the catalog already holds. - return Response(status=http.HTTP_202_ACCEPTED) diff --git a/api/catalog/views/__init__.py b/api/catalog/views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/catalog/views_components.py b/api/catalog/views/components.py similarity index 100% rename from api/catalog/views_components.py rename to api/catalog/views/components.py diff --git a/api/catalog/views/imports.py b/api/catalog/views/imports.py new file mode 100644 index 0000000..ba57ac8 --- /dev/null +++ b/api/catalog/views/imports.py @@ -0,0 +1,56 @@ +import requests +from drf_spectacular.utils import OpenApiResponse, extend_schema +from rest_framework import status as http +from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from rest_framework.views import APIView + +from api.defaults import Throttle +from catalog.models import Service +from catalog.serializers import ImportRequestSerializer, ServiceSerializer +from common.errors import NoStatusPageFound, ProviderUnreachable +from polling.importer import import_from_url + + +class CatalogImportView(APIView): + """Add a service by pasting the address of its status page. + + Its own path, because the body is a URL rather than a service. A + later create that takes a service body then does not overload this + one. + """ + + permission_classes = [AllowAny] + # Every call makes the server fetch a URL somebody else chose. + throttle_scope = Throttle.IMPORT + + @extend_schema( + request=ImportRequestSerializer, + responses={ + 200: ServiceSerializer, + 201: ServiceSerializer, + 400: OpenApiResponse(description="Missing or malformed status_page_url."), + }, + ) + def post(self, request): + body = ImportRequestSerializer(data=request.data) + if not body.is_valid(): + return Response(body.errors, status=400) + url = body.validated_data["status_page_url"] + try: + service, created = import_from_url(url) + except ValueError as error: + # `identify` raises this when no adapter could read the page. + # It is the ordinary outcome of pasting the wrong address, + # and it used to reach the caller as a 500. + raise NoStatusPageFound(str(error)) from error + except requests.RequestException as error: + raise ProviderUnreachable(f"{url} could not be read: {error}") from error + # Read again, after the import. `ServiceSerializer` reads the + # counts `for_display` annotates, and the import returns a + # plain row. + row = Service.objects.for_display(request.user).get(pk=service.pk) + return Response( + ServiceSerializer(row, context={"request": request}).data, + status=http.HTTP_201_CREATED if created else http.HTTP_200_OK, + ) diff --git a/api/catalog/views/requests.py b/api/catalog/views/requests.py new file mode 100644 index 0000000..28ade52 --- /dev/null +++ b/api/catalog/views/requests.py @@ -0,0 +1,56 @@ +from django.db.models import F +from django.utils import timezone +from drf_spectacular.utils import OpenApiResponse, extend_schema +from rest_framework import status as http +from rest_framework.permissions import AllowAny +from rest_framework.response import Response +from rest_framework.views import APIView + +from api.defaults import Throttle +from catalog.models import ServiceRequest, StatusPage +from catalog.serializers import ServiceRequestSerializer + + +class ServiceRequestView(APIView): + """ "Send this URL to us", from the Add-by-URL not-found screen. + + An import stores nothing about an attempt that failed, so this is + where a dead end goes. Telling somebody to hunt for a better link + assumes they have not already tried. + """ + + permission_classes = [AllowAny] + # An anonymous write. Without this one person could inflate the + # demand signal the admin list is ordered by. + # + # Its own scope, not IMPORT's. A failed import and the report that + # follows it share one flow. One bucket would let the import spend + # the report's budget. + throttle_scope = Throttle.SERVICE_REQUEST + + @extend_schema( + request=ServiceRequestSerializer, + responses={ + 202: OpenApiResponse(description="Recorded."), + 400: OpenApiResponse(description="Missing or malformed url."), + }, + ) + def post(self, request): + body = ServiceRequestSerializer(data=request.data) + if not body.is_valid(): + return Response(body.errors, status=400) + url = StatusPage.normalise_url(body.validated_data["url"]) + author = request.user if request.user.is_authenticated else None + row, created = ServiceRequest.objects.get_or_create( + url=url, defaults={"created_by": author, "updated_by": author} + ) + if not created: + # F() rather than a read and a write. Two people asking at + # once would otherwise both write 2. + ServiceRequest.objects.filter(pk=row.pk).update( + request_count=F("request_count") + 1, + last_requested_at=timezone.now(), + ) + # Always 202, and always the same body. Anything else would + # tell a stranger which URLs the catalog already holds. + return Response(status=http.HTTP_202_ACCEPTED) diff --git a/api/catalog/views/services.py b/api/catalog/views/services.py new file mode 100644 index 0000000..3567831 --- /dev/null +++ b/api/catalog/views/services.py @@ -0,0 +1,26 @@ +from rest_framework import generics +from rest_framework.permissions import AllowAny + +from catalog.models import Service +from catalog.serializers import ServiceSerializer +from common.filters import FieldsBackend + + +class ServiceDetailView(generics.RetrieveAPIView): + """The service page's header and its About tab. + + There is no list. Discover searches components, and the signed-out + board lists overall components. So nothing asked a service + collection a question. + """ + + permission_classes = [AllowAny] + lookup_field = "slug" + serializer_class = ServiceSerializer + filter_backends = [FieldsBackend] + # get_queryset reads the request, which schema generation has not + # set. This names the model without running it. + queryset = Service.objects.none() + + def get_queryset(self): + return Service.objects.for_display(self.request.user) diff --git a/api/tests/test_meta.py b/api/tests/test_meta.py index 1e179b7..a08094e 100644 --- a/api/tests/test_meta.py +++ b/api/tests/test_meta.py @@ -216,7 +216,7 @@ def test_every_throttled_view_names_a_rate_that_exists(): from api.defaults import Throttle from authentication.views import MagicLinkView - from catalog.views import CatalogImportView + from catalog.views.imports import CatalogImportView named = {MagicLinkView.throttle_scope, CatalogImportView.throttle_scope} rates = settings.REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] diff --git a/api/tests/test_service_requests.py b/api/tests/test_service_requests.py index 6b26af3..b05e686 100644 --- a/api/tests/test_service_requests.py +++ b/api/tests/test_service_requests.py @@ -103,7 +103,7 @@ def test_a_failed_import_does_not_spend_this_endpoints_budget(client, monkeypatc def refuses(url): raise ValueError(f"no adapter for {url}") - monkeypatch.setattr("catalog.views.import_from_url", refuses) + monkeypatch.setattr("catalog.views.imports.import_from_url", refuses) for _ in range(6): client.post( reverse("catalog-import"), From 4dec33b703c8a88e436d2e3023895f0691616cb9 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 11:45:59 -0700 Subject: [PATCH 39/94] docs: AGENTS.md records where code lives and which docs move The four layout rules were spread across five module docstrings. A reader had to find all five before placing a new expression, and every one of them had to be asked about. The Documentation section read as though every document tracks the code. A plan does not. It records what somebody intended, and rewriting it erases the reasoning a branch followed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index f7d5f6c..0bc2fc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,9 @@ Delete a comment when the reason it recorded stops being true. - No sample output. It goes stale and nobody re-runs it. - No section nobody asked for. - A README describes the repository as it is now, not as it changed. +- A spec describes the system, so it moves with the code. A plan + records what somebody intended, so it stays as written. That includes + the places where the plan turned out to be wrong. ## Engineering @@ -54,6 +57,25 @@ edit that worked. Open the page. Read the value back. template that overrides it. An override patches one instance of a problem; the setting fixes the class. +## Layout + +Which file a piece of code goes in, by what it is. + +- An app keeps one `views.py` until it serves more than one resource. + Then it becomes `views/`, one module per resource. `catalog` serves + three. A sub-resource stays with its parent, so an event's update + log is not a second module. +- A **queryset method** is chainable vocabulary. It lives on the + queryset in `models.py`, because a caller chains it onto a query. +- A **bare expression** is handed to `annotate()`. It lives in + `queries.py`, because nothing chains it. +- `queries.py` is row-scoped and `aggregates.py` is collection-scoped. + One lands in a serialized row. The other lands in the `aggregates` + key beside `results`. +- `common` never imports an app. `common/ordering.py` records this as a + scar. Those expressions lived there, and the base layer imported the + app it describes. + ## Settings The split is by who named the setting. From 6be5a15f5470a14403bc4a9e6db2316d794acc97 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:12:56 -0700 Subject: [PATCH 40/94] refactor: the phase filter's two states are a choices class `PHASE_STATES` was a bare list of tuples in the filter module, the one fixed set outside `status/choices.py`. The word "closed" was written three times, and the labels reached no client. A client read the values off the OpenAPI enum and wrote "Open" and "Closed" itself. `EventPhaseState` names the two values once. `/meta/` publishes the labels beside every other enum, so a rename reaches every screen with no client release. `CLOSED_PHASES` stays on the server, so the client still never restates which phases are terminal. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/common/views.py | 2 ++ api/status/choices.py | 13 +++++++++++++ api/status/filters.py | 8 +++----- api/tests/test_meta.py | 5 +++++ docs/api/openapi.yaml | 9 ++++++++- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/api/common/views.py b/api/common/views.py index 0107c39..fcfd7f2 100644 --- a/api/common/views.py +++ b/api/common/views.py @@ -12,6 +12,7 @@ from status.choices import ( EVENT_PHASES_BY_KIND, EventKind, + EventPhaseState, EventSource, Severity, StatusSource, @@ -62,6 +63,7 @@ def get(self, request): str(kind): _labels(phases) for kind, phases in EVENT_PHASES_BY_KIND.items() }, + "event_phase_state": _labels(EventPhaseState), }, } # `Response(data)` skips `MetaSerializer`, so `?fields=` never diff --git a/api/status/choices.py b/api/status/choices.py index e216ff6..50746ac 100644 --- a/api/status/choices.py +++ b/api/status/choices.py @@ -55,6 +55,19 @@ class EventSource(models.TextChoices): SYSTEM = "system", "Statusboard" +class EventPhaseState(models.TextChoices): + """Whether an event is still running, or over. + + The `phase=` filter takes one of these. `CLOSED_PHASES` below draws + the line and stays on the server, so a client never restates which + phases are terminal. These two labels are what a client shows, so + `/meta/` publishes them beside every other fixed set. + """ + + OPEN = "open", "Open" + CLOSED = "closed", "Closed" + + EVENT_PHASES_BY_KIND = { EventKind.INCIDENT: IncidentPhase, EventKind.MAINTENANCE: MaintenancePhase, diff --git a/api/status/filters.py b/api/status/filters.py index 8ec387e..af40cbb 100644 --- a/api/status/filters.py +++ b/api/status/filters.py @@ -6,17 +6,15 @@ from django_filters import rest_framework as filters -from status.choices import CLOSED_PHASES +from status.choices import CLOSED_PHASES, EventPhaseState from status.models import ServiceEvent -PHASE_STATES = [("open", "Open"), ("closed", "Closed")] - class EventFilter(filters.FilterSet): # Declared, not generated. `CLOSED_PHASES` draws the line between # an open phase and a closed one. A client restating it is a # second copy of one rule. - phase = filters.ChoiceFilter(choices=PHASE_STATES, method="filter_phase") + phase = filters.ChoiceFilter(choices=EventPhaseState.choices, method="filter_phase") service = filters.CharFilter(field_name="service__slug") component = filters.UUIDFilter(field_name="affected_components__id") dashboard = filters.UUIDFilter(method="filter_dashboard") @@ -26,7 +24,7 @@ class Meta: fields = {"kind": ["exact"]} def filter_phase(self, queryset, name, value): - if value == "closed": + if value == EventPhaseState.CLOSED: return queryset.filter(phase__in=CLOSED_PHASES) return queryset.exclude(phase__in=CLOSED_PHASES) diff --git a/api/tests/test_meta.py b/api/tests/test_meta.py index a08094e..849a44f 100644 --- a/api/tests/test_meta.py +++ b/api/tests/test_meta.py @@ -38,6 +38,7 @@ def test_meta_publishes_every_enum(): "event_kind", "event_source", "event_phase", + "event_phase_state", } assert body["enums"]["severity"]["0"] == "Major outage" assert body["enums"]["severity"]["5"] == "Operational" @@ -45,6 +46,9 @@ def test_meta_publishes_every_enum(): assert set(body["enums"]["event_phase"]) == {"incident", "maintenance"} assert "investigating" in body["enums"]["event_phase"]["incident"] assert "scheduled" in body["enums"]["event_phase"]["maintenance"] + # The `phase=` filter's own two values. Without them a client reads + # the values off the OpenAPI enum and writes the labels itself. + assert body["enums"]["event_phase_state"] == {"open": "Open", "closed": "Closed"} @pytest.mark.django_db @@ -84,6 +88,7 @@ def test_fields_enums_with_no_dotted_path_keeps_every_enum(): "event_kind", "event_source", "event_phase", + "event_phase_state", } diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index e4e604c..3248c16 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -356,7 +356,7 @@ paths: - { name: service, in: query, schema: { type: string }, description: Service slug } - { name: component, in: query, schema: { type: string, format: uuid } } - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] } } - - { name: phase, in: query, schema: { type: string, enum: [open, closed] } } + - { name: phase, in: query, schema: { type: string, enum: [open, closed] }, description: 'Render the label from `/meta/` `enums.event_phase_state`.' } - { name: ordering, in: query, schema: { type: string, enum: [-starts_at, starts_at, ends_at, -ends_at], default: -starts_at } } - { $ref: '#/components/parameters/Fields' } - { $ref: '#/components/parameters/Cursor' } @@ -564,6 +564,13 @@ components: description: | Both sets, keyed by kind: `{"incident": {"detected": "Detected", …}, "maintenance": {"scheduled": "Scheduled", …}}` + event_phase_state: + type: object + additionalProperties: { type: string } + description: | + What the `phase` filter on `/events/` takes: + `{"open": "Open", "closed": "Closed"}`. Which phases are terminal stays on + the server, so a client labels the control from here. event_source: type: object additionalProperties: { type: string } From 490d3393f4608ebc00387bdda138bb4553941d11 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:15:28 -0700 Subject: [PATCH 41/94] fix: a multi-select delete in the admin rebuilds the ancestry `ServiceComponent.delete` rebuilds the tree, and `delete_selected` never calls it. It calls `ModelAdmin.delete_queryset` instead, which is the road an admin removing several components actually takes. A deleted middle row left its parent linked to its children, and `parent` is SET_NULL, so the orphan read as a root. The rebuild runs once per service, not once per row, because it rewrites the whole tree either way. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 20 ++++++++++ api/tests/test_admin.py | 81 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index 9de35f7..b32d403 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -35,6 +35,7 @@ from common.queries import related_count from polling.importer import import_from_url from polling.models import Poller, PollRun +from polling.reconcile import rebuild_ancestry, rebuild_search from status.choices import EventKind, Severity from status.models import ComponentStatus, ServiceEvent from status.queries import CURRENT_SEVERITY @@ -584,6 +585,25 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): ).exclude(pk=editing.pk) return super().formfield_for_foreignkey(db_field, request, **kwargs) + def delete_queryset(self, request, queryset): + """Rebuild each service the selection touches, once. + + `delete_selected` is how an admin removes several components, + and it never reaches `ServiceComponent.delete`. Without this a + deleted middle row leaves its parent linked to its children. + `parent` is SET_NULL, so the orphan reads as a root. + + Once per service, not once per row. A rebuild rewrites the whole + tree, so a selection of fifty would otherwise cost fifty passes. + + Read the services before the delete. The rows are gone after it. + """ + services = list(Service.objects.filter(pk__in=queryset.values("service_id"))) + super().delete_queryset(request, queryset) + for service in services: + rebuild_ancestry(service) + rebuild_search(service) + def get_queryset(self, request): return ( super() diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 6a9898f..12b0c0f 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -932,3 +932,84 @@ def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): child.refresh_from_db() assert ancestry(child) == [parent.pk] assert list(ServiceComponent.objects.search("programmable sms")) == [child] + + +def bulk_delete(admin_client, components): + """Run the admin's own delete action, past the confirmation page. + + `delete_selected` is the site-wide action. It calls + `ModelAdmin.delete_queryset`, never the model's `delete`. That is + the road a multi-select delete actually takes. + """ + return admin_client.post( + reverse("admin:catalog_servicecomponent_changelist"), + { + "action": "delete_selected", + "_selected_action": [str(c.pk) for c in components], + "post": "yes", + }, + ) + + +@pytest.mark.django_db +def test_the_bulk_delete_rebuilds_the_ancestry(admin_client): + # `parent` is SET_NULL, so the leaf becomes a root. The cascade + # clears the deleted row's own links only. Without a rebuild the + # grandparent keeps a link to a grandchild it no longer sits above. + from polling.reconcile import rebuild_ancestry + from tests.factories import ComponentFactory, ServiceFactory, ancestry + + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + + bulk_delete(admin_client, [middle]) + + assert ancestry(leaf) == [] + + +@pytest.mark.django_db +def test_the_bulk_delete_rebuilds_the_search_path(admin_client): + # `search_document` holds each ancestor's name. Left stale, a + # deleted parent's name still finds a component under nothing. + from catalog.models import ServiceComponent + from polling.reconcile import rebuild_ancestry, rebuild_search + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory(name="Twilio") + top = ComponentFactory(service=service, name="Programmable Messaging") + leaf = ComponentFactory(service=service, name="SMS", parent=top) + rebuild_ancestry(service) + rebuild_search(service) + + bulk_delete(admin_client, [top]) + + assert list(ServiceComponent.objects.search("messaging")) == [] + # The leaf still answers its own name. A rebuild that dropped the + # whole vector would pass the line above and serve nothing. + assert list(ServiceComponent.objects.search("sms")) == [leaf] + + +@pytest.mark.django_db +def test_the_bulk_delete_rebuilds_a_service_once(admin_client, monkeypatch): + # A rebuild rewrites every row of the service. Once per deleted row + # makes a selection of fifty cost fifty passes over the whole tree. + from catalog import admin as catalog_admin + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory() + top = ComponentFactory(service=service) + first = ComponentFactory(service=service, parent=top) + second = ComponentFactory(service=service, parent=top) + + rebuilt = [] + monkeypatch.setattr( + catalog_admin, "rebuild_ancestry", lambda s: rebuilt.append(s.pk) + ) + monkeypatch.setattr(catalog_admin, "rebuild_search", lambda s: None) + + bulk_delete(admin_client, [first, second]) + + assert rebuilt == [service.pk] From 3916a155734f823b67fe85608b30d95eecb321d6 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:17:14 -0700 Subject: [PATCH 42/94] fix: the admin counts a service's components the way the API does The admin annotated its own count, including the archived rows and the rollup. `Service.component_count` excludes both. One concept had two numbers, and which one you saw depended on the screen. `catalog.queries.component_count` is now the single definition, read by the serializer and by the admin. The View dropdown's link carries the same two filters, because the count sits on the label of that link and has to describe what it opens. Clearing a filter chip still reaches the rollup and the archive. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 16 +++++++++++-- api/catalog/models.py | 7 +++--- api/catalog/queries.py | 15 +++++++++++++ api/tests/test_admin.py | 50 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index b32d403..e88c661 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -19,7 +19,12 @@ from unfold.forms import BaseDialogForm from catalog.models import Service, ServiceComponent, ServiceRequest, StatusPage -from catalog.queries import COMPONENT_WATCHER_COUNT, OVERALL_SEVERITY, is_tracked +from catalog.queries import ( + COMPONENT_WATCHER_COUNT, + OVERALL_SEVERITY, + component_count, + is_tracked, +) from common.admin import ( SEVERITY_VARIANTS, BaseModelAdmin, @@ -355,7 +360,7 @@ def get_queryset(self, request): .select_related("status_page", "poller") .annotate( severity_now=OVERALL_SEVERITY, - component_count=related_count(ServiceComponent.objects, "service"), + component_count=component_count(), event_count=related_count(ServiceEvent.objects, "service"), status_count=related_count( ComponentStatus.objects, "component__service" @@ -395,11 +400,18 @@ def display_related(self, obj): return { "title": _("Related"), "items": [ + # The count is the API's, so the admin and a client + # never disagree about how many parts a service has. + # The link carries the same two filters, because a + # count beside a link describes what the link opens. + # Clearing a chip reaches the rollup and the archive. filtered_list( "admin:catalog_servicecomponent_changelist", _("Components"), obj.component_count, service__id__exact=obj.pk, + is_overall__exact=0, + is_archived__exact=0, ), filtered_list( "admin:status_serviceevent_changelist", diff --git a/api/catalog/models.py b/api/catalog/models.py index fcc80cd..b5a103f 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -26,6 +26,8 @@ def for_display(self, user=None): """ from django.contrib.auth.models import AnonymousUser + from catalog.queries import component_count + tracked = models.Value(0, output_field=models.IntegerField()) if user is not None and not isinstance(user, AnonymousUser): tracked = related_count( @@ -35,10 +37,7 @@ def for_display(self, user=None): return ( self.select_related("status_page", "poller") .annotate( - component_count=related_count( - ServiceComponent.objects.visible().filter(is_overall=False), - "service", - ), + component_count=component_count(), tracked_component_count=tracked, ) .prefetch_related( diff --git a/api/catalog/queries.py b/api/catalog/queries.py index 776f987..d2cd2e8 100644 --- a/api/catalog/queries.py +++ b/api/catalog/queries.py @@ -40,6 +40,21 @@ def is_tracked(field="pk"): return Exists(DashboardItem.objects.filter(component__service=OuterRef(field))) +def component_count(): + """How many components a service publishes, in one query for the page. + + The rollup is the service itself, and an archived row is served + nowhere. So neither is counted, and the number matches the + provider's own page. The API field and the admin column read this, + because two answers to one question drift apart. + """ + from catalog.models import ServiceComponent + + return related_count( + ServiceComponent.objects.visible().filter(is_overall=False), "service" + ) + + def descendant_count(): """How many components sit under each row, in one query for the page. diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 12b0c0f..dea5151 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -1013,3 +1013,53 @@ def test_the_bulk_delete_rebuilds_a_service_once(admin_client, monkeypatch): bulk_delete(admin_client, [first, second]) assert rebuilt == [service.pk] + + +def related_item(admin_user, service, label): + """One entry of a service row's View dropdown, count and link.""" + from django.test import RequestFactory + + from catalog.models import Service + + request = RequestFactory().get("/") + request.user = admin_user + model_admin = admin.site._registry[Service] + row = model_admin.get_queryset(request).get(pk=service.pk) + items = model_admin.display_related(row)["items"] + return next(item for item in items if str(item["title"]).startswith(label)) + + +@pytest.mark.django_db +def test_the_service_row_counts_the_components_the_api_counts(admin_client): + # `Service.component_count` leaves out the rollup and the archived + # rows. A second number under the same name is how the admin and a + # client come to disagree about one service. + from tests.factories import ComponentFactory, ServiceFactory + + admin_user = User.objects.get(email="admin@example.com") + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + ComponentFactory(service=service) + gone = ComponentFactory(service=service) + gone.is_archived = True + gone.save(update_fields=["is_archived"]) + + assert related_item(admin_user, service, "Components")["title"] == "Components (1)" + + +@pytest.mark.django_db +def test_the_components_link_opens_exactly_what_it_counts(admin_client): + # The count sits on the label of a link. A number that does not + # describe the list it opens is worse than no number. + from tests.factories import ComponentFactory, ServiceFactory + + admin_user = User.objects.get(email="admin@example.com") + service = ServiceFactory() + ComponentFactory(service=service, is_overall=True) + ComponentFactory(service=service) + gone = ComponentFactory(service=service) + gone.is_archived = True + gone.save(update_fields=["is_archived"]) + + link = related_item(admin_user, service, "Components")["link"] + assert admin_client.get(link).context["cl"].result_count == 1 From fd33a35c74fb19de89ce01e1284bf7444f336e28 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:22:14 -0700 Subject: [PATCH 43/94] test: the contract and the code agree on response codes `test_contract.py` compared paths and verbs only. That is how the board endpoint came to document a 409 no code path returned, found by hand. A client is generated from the contract and branches on the code, so a failure it cannot see is one it cannot handle. The check compares each documented code against what the view declares, both directions. It found seven disagreements. Three detail endpoints answer 404 and said nothing about it. The board's track annotation sat on `create`, which the generator never reads, so its 200 and 404 reached no schema. Logout answers 200 and an empty object, not 204. Import documented a 422 nothing returns: the real answers are 400 in two shapes, 502 when the provider will not reply, and 429. The request endpoint documented a 429 it did not declare. Magic link now answers 202 rather than 204, which is what the contract has always promised, and its hand-rolled 400 is documented in the shape it really returns. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/authentication/views.py | 8 +++++++- api/catalog/views/components.py | 5 +++++ api/catalog/views/imports.py | 18 +++++++++++++++++- api/catalog/views/requests.py | 4 ++++ api/catalog/views/services.py | 5 +++++ api/dashboards/views.py | 29 +++++++++++++++++------------ api/status/views.py | 7 +++++++ api/tests/test_auth.py | 31 +++++++++++++++++++++++++++++-- api/tests/test_contract.py | 32 +++++++++++++++++++++++++++++++- api/tests/test_import.py | 10 ++++++---- docs/api/openapi.yaml | 32 +++++++++++++++++++++++++++++--- 11 files changed, 157 insertions(+), 24 deletions(-) diff --git a/api/authentication/views.py b/api/authentication/views.py index 06d5192..364b8cc 100644 --- a/api/authentication/views.py +++ b/api/authentication/views.py @@ -35,6 +35,10 @@ class MagicLinkView(APIView): request=MagicLinkRequestSerializer, responses={ 202: OpenApiResponse(description="Sent, if the address can receive it."), + # Hand rolled, so it is a bare `detail` and carries no + # `code`. A client generated from the contract could not + # see this failure at all. + 400: OpenApiResponse(description="No email address in the body."), 429: OpenApiResponse(description="Too many requests."), }, ) @@ -47,7 +51,9 @@ def post(self, request): user, _ = User.objects.get_or_create(email=email) link = MagicLinkToken.objects.create(user=user) send_magic_link(link) - return Response(status=status.HTTP_204_NO_CONTENT) + # Accepted, not No content. We say nothing about whether the + # address can receive mail, and the contract promises 202. + return Response(status=status.HTTP_202_ACCEPTED) class VerifyView(APIView): diff --git a/api/catalog/views/components.py b/api/catalog/views/components.py index e005c56..ef2382c 100644 --- a/api/catalog/views/components.py +++ b/api/catalog/views/components.py @@ -6,6 +6,7 @@ """ from django_filters.rest_framework import DjangoFilterBackend +from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import generics from rest_framework.permissions import AllowAny @@ -16,6 +17,7 @@ from common.aggregates import StatusAggregateSet from common.filters import FieldsBackend from common.ordering import MappedOrderingFilter +from common.serializers import ErrorSerializer from status.queries import CURRENT_SEVERITY @@ -82,6 +84,9 @@ def filter_queryset(self, queryset): return super().filter_queryset(queryset) +@extend_schema_view( + get=extend_schema(responses={200: ComponentSerializer, 404: ErrorSerializer}) +) class ComponentDetailView(ComponentQueryMixin, generics.RetrieveAPIView): filter_backends = [FieldsBackend] lookup_field = "pk" diff --git a/api/catalog/views/imports.py b/api/catalog/views/imports.py index ba57ac8..ad082e7 100644 --- a/api/catalog/views/imports.py +++ b/api/catalog/views/imports.py @@ -9,6 +9,7 @@ from catalog.models import Service from catalog.serializers import ImportRequestSerializer, ServiceSerializer from common.errors import NoStatusPageFound, ProviderUnreachable +from common.serializers import ErrorSerializer from polling.importer import import_from_url @@ -29,7 +30,22 @@ class CatalogImportView(APIView): responses={ 200: ServiceSerializer, 201: ServiceSerializer, - 400: OpenApiResponse(description="Missing or malformed status_page_url."), + # Two shapes under one code. A malformed body answers the + # serializer's field map. A URL no adapter can read answers + # the named error. The contract carries both. + 400: OpenApiResponse( + description=( + "Missing or malformed status_page_url, or no status page " + "found at it." + ) + ), + 429: OpenApiResponse( + response=ErrorSerializer, description="Too many requests." + ), + 502: OpenApiResponse( + response=ErrorSerializer, + description="The provider could not be reached.", + ), }, ) def post(self, request): diff --git a/api/catalog/views/requests.py b/api/catalog/views/requests.py index 28ade52..9180f93 100644 --- a/api/catalog/views/requests.py +++ b/api/catalog/views/requests.py @@ -9,6 +9,7 @@ from api.defaults import Throttle from catalog.models import ServiceRequest, StatusPage from catalog.serializers import ServiceRequestSerializer +from common.serializers import ErrorSerializer class ServiceRequestView(APIView): @@ -33,6 +34,9 @@ class ServiceRequestView(APIView): responses={ 202: OpenApiResponse(description="Recorded."), 400: OpenApiResponse(description="Missing or malformed url."), + 429: OpenApiResponse( + response=ErrorSerializer, description="Too many requests." + ), }, ) def post(self, request): diff --git a/api/catalog/views/services.py b/api/catalog/views/services.py index 3567831..5736fa3 100644 --- a/api/catalog/views/services.py +++ b/api/catalog/views/services.py @@ -1,11 +1,16 @@ +from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import generics from rest_framework.permissions import AllowAny from catalog.models import Service from catalog.serializers import ServiceSerializer from common.filters import FieldsBackend +from common.serializers import ErrorSerializer +@extend_schema_view( + get=extend_schema(responses={200: ServiceSerializer, 404: ErrorSerializer}) +) class ServiceDetailView(generics.RetrieveAPIView): """The service page's header and its About tab. diff --git a/api/dashboards/views.py b/api/dashboards/views.py index 5520856..42ac427 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -1,6 +1,6 @@ from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import OpenApiResponse, extend_schema +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view from rest_framework import generics, status from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response @@ -22,6 +22,22 @@ def _board(request, uuid): return get_object_or_404(Dashboard, id=uuid, owner=request.user) +# On `post`, not on `create`. The generator reads the handler the router +# dispatches to. An annotation on `create` was dropped in silence, so +# the documented 200 and 404 reached no schema. +@extend_schema_view( + post=extend_schema( + request=TrackComponentSerializer, + responses={ + 201: ComponentSerializer, + 200: OpenApiResponse( + response=ComponentSerializer, + description="Already tracked. Tracking twice is not an error.", + ), + 404: OpenApiResponse(description="No such board, or no such component."), + }, + ) +) class BoardComponentListView(generics.ListCreateAPIView): # Schema generation reads the model off this before it calls # get_queryset. It names the model without running one. @@ -54,17 +70,6 @@ def get_queryset(self): .distinct() ) - @extend_schema( - request=TrackComponentSerializer, - responses={ - 201: ComponentSerializer, - 200: OpenApiResponse( - response=ComponentSerializer, - description="Already tracked. Tracking twice is not an error.", - ), - 404: OpenApiResponse(description="No such board, or no such component."), - }, - ) def create(self, request, *args, **kwargs): board = _board(request, self.kwargs["uuid"]) # `visible`, so a write agrees with the reads. An archived id diff --git a/api/status/views.py b/api/status/views.py index ec80b6a..acd8d49 100644 --- a/api/status/views.py +++ b/api/status/views.py @@ -9,6 +9,7 @@ from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend +from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import generics from rest_framework.exceptions import NotAuthenticated from rest_framework.permissions import AllowAny @@ -17,6 +18,7 @@ from common.filters import FieldsBackend from common.ordering import MappedOrderingFilter from common.pagination import TimelinePagination +from common.serializers import ErrorSerializer from dashboards.models import Dashboard from status.filters import EventFilter from status.models import EventUpdate, ServiceEvent @@ -63,6 +65,11 @@ def get_queryset(self): return queryset +@extend_schema_view( + get=extend_schema( + responses={200: ServiceEventDetailSerializer, 404: ErrorSerializer} + ) +) class EventDetailView(generics.RetrieveAPIView): permission_classes = [AllowAny] serializer_class = ServiceEventDetailSerializer diff --git a/api/tests/test_auth.py b/api/tests/test_auth.py index 243677d..9c0f27a 100644 --- a/api/tests/test_auth.py +++ b/api/tests/test_auth.py @@ -22,7 +22,9 @@ def test_requesting_a_link_sends_one_and_creates_a_token(): response = APIClient().post( reverse("magic-link"), {"email": "a@b.com"}, format="json" ) - assert response.status_code == 204 + # 202, not 204. We accept the address and say nothing about whether + # it can receive mail, which is what the contract promises. + assert response.status_code == 202 assert len(mail.outbox) == 1 assert MagicLinkToken.objects.filter(user__email="a@b.com").count() == 1 @@ -36,7 +38,7 @@ def test_requesting_a_link_for_an_unknown_email_looks_identical(): second = APIClient().post( reverse("magic-link"), {"email": "new@b.com"}, format="json" ) - assert first.status_code == second.status_code == 204 + assert first.status_code == second.status_code == 202 @pytest.mark.django_db @@ -188,3 +190,28 @@ def test_the_sign_in_page_is_not_the_endpoint_it_calls(): from django.urls import reverse assert settings.APP_MAGIC_LINK_PATH != reverse("verify") + + +@pytest.mark.django_db +def test_a_link_request_with_no_email_says_which_shape_it_refuses_in(): + # Hand rolled, so it is a bare `detail` with no `code`. The contract + # documents that shape rather than claiming the `Error` one. + response = APIClient().post(reverse("magic-link"), {}, format="json") + assert response.status_code == 400 + assert response.json() == {"detail": "email is required"} + + +@pytest.mark.django_db +def test_logging_out_answers_with_an_empty_object(): + # simplejwt's view answers 200 and a body, not 204. The contract + # documents what ships, because a client parses the response. + APIClient().post(reverse("magic-link"), {"email": "a@b.com"}, format="json") + token = MagicLinkToken.objects.get().token + pair = APIClient().post(reverse("verify"), {"token": token}, format="json").json() + + response = APIClient().post( + reverse("logout"), {"refresh": pair["refresh"]}, format="json" + ) + + assert response.status_code == 200 + assert response.json() == {} diff --git a/api/tests/test_contract.py b/api/tests/test_contract.py index 7aac92d..6564850 100644 --- a/api/tests/test_contract.py +++ b/api/tests/test_contract.py @@ -28,12 +28,25 @@ def generated(): return SchemaGenerator().get_schema(request=None, public=True) +VERBS = ("get", "post", "delete", "patch", "put") + + def _paths(schema): return { f"{verb.upper()} {path}" for path, ops in schema["paths"].items() for verb in ops - if verb in ("get", "post", "delete", "patch", "put") + if verb in VERBS + } + + +def _responses(schema): + return { + f"{verb.upper()} {path} -> {code}" + for path, ops in schema["paths"].items() + for verb, operation in ops.items() + if verb in VERBS + for code in operation.get("responses", {}) } @@ -47,6 +60,23 @@ def test_no_operation_exists_that_the_contract_does_not_document(committed, gene assert not extra, f"implemented but undocumented: {sorted(extra)}" +def test_every_documented_response_code_is_declared_in_the_code(committed, generated): + # The board endpoint documented a 409 no code path returned. The + # path check above passes on that, because the path exists. A + # client is generated from this file and branches on the code. + missing = _responses(committed) - _responses(generated) + assert not missing, f"documented but not returned: {sorted(missing)}" + + +def test_no_response_code_is_declared_that_the_contract_does_not_document( + committed, generated +): + # The other direction. A failure a client cannot see is one it + # cannot handle, so it falls through as an unexpected error. + extra = _responses(generated) - _responses(committed) + assert not extra, f"returned but undocumented: {sorted(extra)}" + + def test_every_endpoint_accepts_the_fields_parameter(generated): # ?fields= is baked in at the base layer, so it must appear on every # operation without a per-view annotation. diff --git a/api/tests/test_import.py b/api/tests/test_import.py index b590672..b24edb8 100644 --- a/api/tests/test_import.py +++ b/api/tests/test_import.py @@ -120,10 +120,12 @@ def test_the_response_carries_the_detected_provider(): @pytest.mark.django_db def test_a_missing_status_page_url_is_a_400(): - assert ( - APIClient().post(reverse("catalog-import"), {}, format="json").status_code - == 400 - ) + response = APIClient().post(reverse("catalog-import"), {}, format="json") + assert response.status_code == 400 + # The serializer's own field map, not the `Error` shape. The + # contract documents both under this code, and a client tells them + # apart by whether `code` is there. + assert response.json() == {"status_page_url": ["This field is required."]} def test_normalising_keeps_the_www_prefix(): diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 3248c16..28c288d 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -161,6 +161,13 @@ paths: schema: { type: object, required: [email], properties: { email: { type: string, format: email } } } responses: '202': { description: Accepted } + '400': + description: | + No email address in the body. A bare `detail` string, hand rolled here — not the + `Error` shape, so there is no `code` to branch on. + content: + application/json: + schema: { type: object, properties: { detail: { type: string } } } '429': { $ref: '#/components/responses/Throttled' } /auth/verify/: post: @@ -190,7 +197,10 @@ paths: post: tags: [auth] summary: Blacklist the refresh token - responses: { '204': { description: No content } } + responses: + '200': + description: Blacklisted. The body is an empty object. + content: { application/json: { schema: { type: object } } } /me/: get: @@ -236,8 +246,24 @@ paths: '201': description: Service created content: { application/json: { schema: { $ref: '#/components/schemas/Service' } } } - '422': - description: No supported status page found at that URL + '400': + description: | + Two failures share this code. A missing or malformed `status_page_url` answers a + field-errors object, one array of messages per field name. A URL no adapter can + read answers the `Error` shape with `code: no_status_page_found`. Branch on + whether `code` is present. + content: + application/json: + schema: + oneOf: + - type: object + additionalProperties: { type: array, items: { type: string } } + - $ref: '#/components/schemas/Error' + '429': { $ref: '#/components/responses/Throttled' } + '502': + description: | + The provider did not answer. `code: provider_unreachable`. The URL may still be a + status page, so this is worth retrying and `400` is not. content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } /catalog/requests/: From c38047e616dee2f9b04a2f2e8bc43a22d29cb37d Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:24:18 -0700 Subject: [PATCH 44/94] fix: a component's path names only ancestors that are served Each path node carries an id so a client can link to it, and the component detail answers 404 for an archived row. The walk had no archive filter, so a breadcrumb's middle step linked to a page nothing serves. An archived step is left out rather than marked. Every other reader answers "not served" by dropping the row, and a marker node would carry an id nothing can fetch. The chain is then shorter than the real tree and its last node is not always `parent`, which the contract now says. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/models.py | 14 ++++++++++++++ api/catalog/serializers.py | 5 ++++- api/tests/test_component_api.py | 19 +++++++++++++++++++ docs/api/openapi.yaml | 14 +++++++++----- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/api/catalog/models.py b/api/catalog/models.py index b5a103f..fc75ff8 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -525,6 +525,20 @@ def ancestors(self): node = node.parent return list(reversed(chain)) + @property + def visible_ancestors(self): + """The ancestors a caller is served, top down. + + `ServiceComponentQuerySet.visible` is the same rule in SQL. + `ancestors` has already loaded these rows, so asking the + database again would cost a query a row. + + A dropped step makes the chain shorter, never broken. Every + other reader answers "not served" by leaving the row out. A + marker node would carry an id nothing can fetch. + """ + return [row for row in self.ancestors if not row.is_archived] + @property def path(self): """Where the component sits, read from the service down.""" diff --git a/api/catalog/serializers.py b/api/catalog/serializers.py index 40e4712..36396fe 100644 --- a/api/catalog/serializers.py +++ b/api/catalog/serializers.py @@ -103,10 +103,13 @@ def get_path(self, row): # `ancestors` walks the chain and guards the loop a self # referencing column allows. This walked it a second time and # did not. + # + # Archived steps are left out. Each node carries an id a client + # links to, and an archived component answers 404. if row.is_overall: return [] return PathNodeSerializer( - row.ancestors, + row.visible_ancestors, many=True, context=self.context, fields_tree=self.child_tree("path"), diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py index 17afaee..654c941 100644 --- a/api/tests/test_component_api.py +++ b/api/tests/test_component_api.py @@ -220,3 +220,22 @@ def test_a_ranked_search_pages_without_repeating_a_row(client): assert seen == list(dict.fromkeys(seen)), "a row came back on two pages" assert len(seen) == 7, "a row was never reached" + + +def test_the_path_leaves_out_an_archived_ancestor(client): + # Each node carries an id a client links to, and the detail answers + # 404 for an archived component. A breadcrumb step pointing at one + # is a link to a page nothing serves. + service = ServiceFactory() + top = ComponentFactory(service=service, name="Messaging") + middle = ComponentFactory(service=service, name="Programmable", parent=top) + leaf = ComponentFactory(service=service, name="SMS", parent=middle) + middle.is_archived = True + middle.save(update_fields=["is_archived"]) + + body = client.get(reverse("component-detail", args=[leaf.id])).json() + + assert [node["name"] for node in body["path"]] == ["Messaging"] + # Shorter, not broken. `parent` is the raw column and still names + # the archived row, so the two disagree by design. + assert body["parent"] == str(middle.id) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 28c288d..b4e4d26 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -26,8 +26,9 @@ info: Three consequences follow. A tracked component that is archived leaves the board with no notice. An event names the components it affected, and a link to an archived one answers - `404`. `Component.path` and `Component.parent` name ancestors without checking them, so a - breadcrumb step can answer `404` too. Render a node you cannot fetch as plain text. + `404`. `Component.path` lists the ancestors that are served, so a breadcrumb can skip a + level, and its last node is not always `Component.parent`. `parent` is the raw column and + still names an archived row, so fetch a breadcrumb from `path`. ## Severity is the state @@ -873,9 +874,12 @@ components: path: type: array description: | - Every component above this one, top down. Each step carries its own id, so a client - builds a breadcrumb it can click. Empty on the overall component, which is under - nothing. + Every component above this one that is served, top down. Each step carries its own + id, so a client builds a breadcrumb it can click and every node here is fetchable. + Empty on the overall component, which is under nothing. + + An archived ancestor is left out. The chain is then shorter than the real tree, and + its last node is not `parent`. It is never broken, so nothing needs a placeholder. items: { $ref: '#/components/schemas/PathNode' } parent: { type: string, format: uuid, nullable: true } descendant_count: From 036bf9bab31cd6f62add023a84625a0c484fc906 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:28:00 -0700 Subject: [PATCH 45/94] test: every reader that serves components excludes the archived ones An archived component is served nowhere, and three readers have forgotten it: the Affects badge, `tracked_component_count` and the board write. Each was caught by hand in review. Remembering to call `visible()` has failed three times, so the suite asks instead. One probe per reader, each archiving a component and reporting what a caller can still see of it. A coverage check walks the contract's refs and fails by path when an operation that can return a component has no probe, the way the `?fields=` probes already do. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_archived.py | 251 +++++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 api/tests/test_archived.py diff --git a/api/tests/test_archived.py b/api/tests/test_archived.py new file mode 100644 index 0000000..2fe14f7 --- /dev/null +++ b/api/tests/test_archived.py @@ -0,0 +1,251 @@ +"""The archived rule, against every reader that serves components. + +A provider stops publishing a component. The row is archived rather +than deleted, because somebody may be tracking it. +`ServiceComponentQuerySet.visible` is the one definition of which rows +a caller is served. + +Three readers have forgotten to call it: the Affects badge, +`tracked_component_count` and the board write. Each was caught by hand +in review. Remembering has failed three times, so the suite asks +instead. +""" + +from pathlib import Path +from unittest import mock + +import pytest +import yaml +from django.urls import reverse +from django.utils import timezone +from rest_framework.test import APIClient + +from authentication.models import User +from dashboards.models import DashboardItem +from polling.reconcile import rebuild_ancestry +from status.choices import EventKind, IncidentPhase +from status.models import ServiceEvent +from tests.conftest import jwt_client +from tests.factories import ComponentFactory, ServiceFactory, StatusPageFactory + +CONTRACT = Path(__file__).resolve().parents[2] / "docs" / "api" / "openapi.yaml" + +pytestmark = pytest.mark.django_db + + +def archived(service=None, **fields): + """A component the provider stopped publishing.""" + row = ComponentFactory(service=service or ServiceFactory(), **fields) + row.is_archived = True + row.save(update_fields=["is_archived"]) + return row + + +def signed_in(): + return jwt_client(User.objects.create(email="archived-probe@example.test")) + + +def incident(service): + return ServiceEvent.objects.create( + service=service, + external_id="probe", + kind=EventKind.INCIDENT, + title="x", + phase=IncidentPhase.DETECTED, + starts_at=timezone.now(), + ) + + +# Each probe archives one component and returns what a caller can still +# see of it. Anything truthy is a leak. + + +def component_list(): + gone = archived() + body = APIClient().get(reverse("component-list"), {"service": gone.service.slug}) + return body.json()["results"] or body.json()["aggregates"]["total"] + + +def descendants_of_a_component(): + service = ServiceFactory() + top = ComponentFactory(service=service) + archived(service=service, parent=top) + rebuild_ancestry(service) + url = reverse("component-list") + return APIClient().get(url, {"ancestor": str(top.id)}).json()["results"] + + +def an_events_affected_list(): + gone = archived() + event = incident(gone.service) + event.affected_components.add(gone) + url = reverse("component-list") + return APIClient().get(url, {"event": str(event.id)}).json()["results"] + + +def component_detail(): + gone = archived() + response = APIClient().get(reverse("component-detail", args=[gone.id])) + return response.status_code != 404 + + +def a_components_path(): + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = archived(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + rebuild_ancestry(service) + body = APIClient().get(reverse("component-detail", args=[leaf.id])).json() + return [node for node in body["path"] if node["id"] == str(middle.id)] + + +def a_components_descendant_count(): + service = ServiceFactory() + top = ComponentFactory(service=service) + archived(service=service, parent=top) + rebuild_ancestry(service) + body = APIClient().get(reverse("component-detail", args=[top.id])).json() + return body["descendant_count"] + + +def a_services_component_count(): + service = ServiceFactory() + StatusPageFactory(service=service) + archived(service=service) + url = reverse("service-detail", kwargs={"slug": service.slug}) + return APIClient().get(url).json()["component_count"] + + +def a_services_tracked_count(): + service = ServiceFactory() + StatusPageFactory(service=service) + gone = archived(service=service) + caller = signed_in() + DashboardItem.objects.create( + dashboard=User.objects.get( + email="archived-probe@example.test" + ).default_dashboard, + component=gone, + ) + url = reverse("service-detail", kwargs={"slug": service.slug}) + return caller.get(url).json()["tracked_component_count"] + + +def an_imported_services_counts(): + service = ServiceFactory() + StatusPageFactory(service=service) + archived(service=service) + with ( + mock.patch("polling.fetch.check", lambda url: url), + mock.patch( + "catalog.views.imports.import_from_url", return_value=(service, False) + ), + ): + body = APIClient().post( + reverse("catalog-import"), + {"status_page_url": "https://status.example.com/"}, + format="json", + ) + return body.json()["component_count"] + + +def the_board_list(): + caller = signed_in() + board = User.objects.get(email="archived-probe@example.test").default_dashboard + gone = archived() + DashboardItem.objects.create(dashboard=board, component=gone) + url = reverse("board-components", kwargs={"uuid": board.id}) + body = caller.get(url).json() + return body["results"] or body["aggregates"]["total"] + + +def the_track_write(): + caller = signed_in() + board = User.objects.get(email="archived-probe@example.test").default_dashboard + gone = archived() + url = reverse("board-components", kwargs={"uuid": board.id}) + response = caller.post(url, {"component_id": str(gone.id)}, format="json") + return response.status_code != 404 + + +def the_affects_badge(): + gone = archived() + event = incident(gone.service) + event.affected_components.add(gone) + url = reverse("event-detail", kwargs={"uuid": event.id}) + return APIClient().get(url).json()["affected_count"] + + +# One entry per reader, against the contract path it answers on. The +# path is what the coverage check below reads. +ARCHIVED_PROBES = { + "the component list": ("/catalog/components/", component_list), + "a component's descendants": ("/catalog/components/", descendants_of_a_component), + "an event's affected list": ("/catalog/components/", an_events_affected_list), + "the component detail": ("/catalog/components/{uuid}/", component_detail), + "a component's path": ("/catalog/components/{uuid}/", a_components_path), + "a component's descendant count": ( + "/catalog/components/{uuid}/", + a_components_descendant_count, + ), + "a service's component count": ( + "/catalog/services/{slug}/", + a_services_component_count, + ), + "a service's tracked count": ( + "/catalog/services/{slug}/", + a_services_tracked_count, + ), + "an imported service's counts": ("/catalog/import/", an_imported_services_counts), + "the board list": ("/dashboards/{uuid}/components/", the_board_list), + "the track write": ("/dashboards/{uuid}/components/", the_track_write), + "the Affects badge": ("/events/{uuid}/", the_affects_badge), +} + + +@pytest.mark.parametrize("reader", sorted(ARCHIVED_PROBES)) +def test_no_reader_serves_an_archived_component(reader): + # The one exception is `Service.overall_component`. A service whose + # rollup is archived still needs a status in its header, and + # `test_catalog_api` covers that. + _path, probe = ARCHIVED_PROBES[reader] + leaked = probe() + assert not leaked, f"{reader} served an archived component: {leaked}" + + +def _reachable(node, schemas, found=None): + """Every schema an operation can produce, following each `$ref`.""" + found = set() if found is None else found + if isinstance(node, list): + for item in node: + _reachable(item, schemas, found) + return found + if not isinstance(node, dict): + return found + for key, value in node.items(): + if key == "$ref" and value.startswith("#/components/schemas/"): + name = value.rsplit("/", 1)[1] + if name not in found: + found.add(name) + _reachable(schemas.get(name, {}), schemas, found) + else: + _reachable(value, schemas, found) + return found + + +def test_the_probes_cover_every_operation_that_can_serve_a_component(): + # A reader added later that forgets `visible()` serves a row the + # provider stopped publishing, and nothing says so. This fails by + # path, the way the `?fields=` probes do. + contract = yaml.safe_load(CONTRACT.read_text()) + schemas = contract["components"]["schemas"] + serves = { + path + for path, operations in contract["paths"].items() + for operation in operations.values() + if "Component" in _reachable(operation.get("responses", {}), schemas) + } + probed = {path for path, _probe in ARCHIVED_PROBES.values()} + assert not serves - probed, ( + f"serves components, unprobed: {sorted(serves - probed)}" + ) From a3bfc80766947ebfb1b9c86dc8f722f209971b1d Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:29:38 -0700 Subject: [PATCH 46/94] test: a column declaring two defaults declares one value `ServiceEvent.detected_by`, `EventUpdate.source` and `ServiceComponent.is_archived` each carry `default` and `db_default`. Both are needed: one fills an unsaved instance, the other fills a column a writer omits. Nothing held them together. If the two ever disagreed, a row Python made and a row SQL made would differ on the same column in silence. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_common_models.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/api/tests/test_common_models.py b/api/tests/test_common_models.py index e1dfbfe..746e36d 100644 --- a/api/tests/test_common_models.py +++ b/api/tests/test_common_models.py @@ -53,3 +53,30 @@ def test_the_system_account_exists_before_anything_writes(): assert account.is_active is False # It signed itself, so no row the system wrote is left unsigned. assert account.created_by_id == account.pk + + +def test_a_column_with_two_defaults_declares_one_value(): + """`default` and `db_default` do different jobs, on the same column. + + Without `default` an unsaved instance holds a sentinel rather than + the value. Without `db_default` the column carries no default, and + a writer that predates it fails the insert. + + A field needs both. If the two ever disagree, a row Python makes + and a row SQL makes differ on one column. Nothing would say so. + """ + from django.apps import apps + from django.db.models import NOT_PROVIDED + + both = { + f"{model._meta.label}.{field.name}": (field.default, field.db_default) + for model in apps.get_models() + for field in model._meta.concrete_fields + if field.default is not NOT_PROVIDED and field.db_default is not NOT_PROVIDED + } + # Nothing declaring both would make the check below vacuous. + assert both + disagreeing = { + name: values for name, values in both.items() if values[0] != values[1] + } + assert not disagreeing, f"two defaults, two values: {disagreeing}" From 0412aed81f336f4fa05a9da4432c09afaa2a4df3 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:31:09 -0700 Subject: [PATCH 47/94] docs: featuring is on any component, and only on the component admin The comment on `is_featured` described the flag as belonging to a service's overall component, which stopped being true when any component became featurable. It also claimed the flag is the whole sort on day one, which was never true: severity is the second key and breaks the tie whatever the watcher counts are. The admin help text carried the same stale instruction. The spec still described an inline on the service admin that no longer exists, and a spec that contradicts the code is worse than none. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 5 +++-- api/catalog/models.py | 7 ++++--- docs/specs/2026-09-03-statusboard-client-design.md | 11 +++++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index e88c661..7547220 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -562,8 +562,9 @@ class ServiceComponentAdmin( { "fields": ["is_featured"], "description": _( - "Leads the suggested sort on the catalog. Tick it on a " - "service's overall component to surface the service." + "Leads the suggested sort on the catalog. Tick it on any " + "component. A rollup surfaces its service, and a leaf " + "surfaces that part." ), }, ), diff --git a/api/catalog/models.py b/api/catalog/models.py index fc75ff8..1c5f16d 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -336,9 +336,10 @@ class ServiceComponent(BaseModel): status_page_order = models.IntegerField(verbose_name="Page order", default=0) is_overall = models.BooleanField(verbose_name="Overall", default=False) - # Ticked on the overall component to feature a whole service. It is - # the first key of the suggested sort. On day one that is the whole - # sort: every watcher count starts at zero. + # Ticked on any component. The suggested sort leads with it. A + # featured rollup surfaces its service, and a featured leaf + # surfaces that part. Severity is the next key, so it breaks the + # tie among the featured rows. is_featured = models.BooleanField(verbose_name="Featured", default=False) # Archiving is the flag. The date is kept because `updated_at` # moves on every save, so a later rename would erase it. `save` sets diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md index ed766e8..fd58de3 100644 --- a/docs/specs/2026-09-03-statusboard-client-design.md +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -341,10 +341,13 @@ The service version needed three. ### Admin -`ServiceComponent.is_featured` is how a service is featured, ticked on its overall -component. So the component admin gains that column, a list filter and a bulk action. The -service admin edits its overall component's flag inline, because "feature this service" is -the question an admin actually has. +`ServiceComponent.is_featured` is how anything is featured, and any component may carry +it. A featured rollup surfaces its service. A featured leaf surfaces that part. + +Featuring is viewed and managed on the component admin, and nowhere else. That admin gains +the column, a list filter, the field on the change form, and two bulk actions for a +selection. The service admin has no featuring control: one flag edited from two screens is +one question with two answers. `ServiceComponent.watcher_count` joins the same list, read-only. From eb3d42204b89d4f9fbbcb2a4c9377752dda6f427 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:56:27 -0700 Subject: [PATCH 48/94] fix: the ancestry backfill recomputes from parent The backfill read `ancestor_ids`, which could name a deleted component. It dropped that step and kept the steps above it. `parent` is SET_NULL, so such a row is a root and has no ancestors at all. The table then answered `?ancestor=` with a tree nobody sits in, and inflated a descendant count. Only a poll repaired it, and an untracked service is never polled again. `rebuild_ancestry` defines what a component's ancestry is, so the backfill now does its walk, with the same cycle and service guards. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- .../0008_component_ancestor_table.py | 52 +++++----- api/tests/test_migrations.py | 96 +++++++++++++++++++ 2 files changed, 121 insertions(+), 27 deletions(-) create mode 100644 api/tests/test_migrations.py diff --git a/api/catalog/migrations/0008_component_ancestor_table.py b/api/catalog/migrations/0008_component_ancestor_table.py index 45c3285..86542aa 100644 --- a/api/catalog/migrations/0008_component_ancestor_table.py +++ b/api/catalog/migrations/0008_component_ancestor_table.py @@ -6,37 +6,39 @@ from django.db import migrations, models -def fill_from_ancestor_ids(apps, schema_editor): - """Copy the array into the table, before the column goes. +def fill_from_parent(apps, schema_editor): + """Fill the table with the walk `rebuild_ancestry` does. - Migration 0004 backfilled the array with a recursive walk of - `parent`. Reading it again here makes the two agree by - construction. Recomputing would be a second answer to the same - question. + That walk defines a component's ancestry, so the table starts where + a poll would leave it. The dropped array is not read. It kept the + steps above a deleted component, and that is the defect this table + removes. - The array can name a component somebody deleted. A foreign key - cannot, so such a step is dropped. That dangling reference is the - defect this table removes. + `ServiceComponent.ancestors` holds the rules the walk obeys. Python, not SQL, because the key defaults to `uuid.uuid7`. Postgres 17 has no function for version 7. """ ServiceComponent = apps.get_model("catalog", "ServiceComponent") ComponentAncestor = apps.get_model("catalog", "ComponentAncestor") - live = set(ServiceComponent.objects.values_list("id", flat=True)) - links = [ - ComponentAncestor( - ancestor_id=up, - descendant_id=row_id, - # The array reads root first, so the last step is the parent. - depth=len(ancestors) - step, + rows = { + row_id: (parent_id, service_id) + for row_id, parent_id, service_id in ServiceComponent.objects.values_list( + "id", "parent_id", "service_id" ) - for row_id, ancestors in ServiceComponent.objects.values_list( - "id", "ancestor_ids" - ) - for step, up in enumerate(ancestors) - if up in live - ] + } + links = [] + for row_id, (parent_id, service_id) in rows.items(): + chain, seen, up = [], {row_id}, parent_id + while up in rows and up not in seen and rows[up][1] == service_id: + seen.add(up) + chain.append(up) + up = rows[up][0] + links += [ + ComponentAncestor(ancestor_id=up, descendant_id=row_id, depth=depth) + # The walk climbs, so the first step is the parent. + for depth, up in enumerate(chain, start=1) + ] ComponentAncestor.objects.bulk_create(links, batch_size=1000) @@ -103,11 +105,7 @@ class Migration(migrations.Migration): name="ancestry_depth_starts_at_one", ), ), - # The read comes before the drop. The column is the only record - # of the tree that survives a service nobody polls. - migrations.RunPython( - fill_from_ancestor_ids, reverse_code=migrations.RunPython.noop - ), + migrations.RunPython(fill_from_parent, reverse_code=migrations.RunPython.noop), migrations.RemoveIndex( model_name="servicecomponent", name="components_by_ancestor", diff --git a/api/tests/test_migrations.py b/api/tests/test_migrations.py new file mode 100644 index 0000000..0acddb4 --- /dev/null +++ b/api/tests/test_migrations.py @@ -0,0 +1,96 @@ +"""The data migration that fills the ancestry table. + +It runs once. A service nobody polls is never rebuilt, so what this +writes is what that service keeps. + +The function is called directly, against the live app registry. It +reads `parent`, which the models still hold, so a historical registry +would answer the same. +""" + +from importlib import import_module + +import pytest + +from catalog.models import ComponentAncestor +from tests.factories import ComponentFactory, ServiceFactory, ancestry + +fill_from_parent = import_module( + "catalog.migrations.0008_component_ancestor_table" +).fill_from_parent + +pytestmark = pytest.mark.django_db + + +def backfill(): + """Run the migration against a table it has not seen.""" + from django.apps import apps + + ComponentAncestor.objects.all().delete() + fill_from_parent(apps, None) + + +def test_the_backfill_writes_the_chain_of_every_component(): + # Nothing else fills the table on a service nobody polls. An empty + # backfill leaves every such component with no descendants and no + # breadcrumb, forever. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + + backfill() + + assert ancestry(top) == [] + assert ancestry(middle) == [top.id] + assert ancestry(leaf) == [top.id, middle.id] + + +def test_the_backfill_gives_a_row_whose_parent_went_no_ancestors(): + # `parent` is SET_NULL, so deleting a component makes its children + # roots. Keeping the steps above the gap answers `?ancestor=` with a + # tree nobody sits in. It also inflates the top row's count. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + leaf = ComponentFactory(service=service, parent=middle) + middle.delete() + + backfill() + + leaf.refresh_from_db() + assert ancestry(leaf) == [] + assert top.descendant_links.count() == 0 + + +def test_the_backfill_stops_at_the_service_boundary(): + # Moving a component to another service leaves its old children + # pointing across. A step there names a row the old service's lists + # never hold. + old = ServiceFactory() + top = ComponentFactory(service=old) + leaf = ComponentFactory(service=old, parent=top) + top.service = ServiceFactory() + top.save() + + backfill() + + assert ancestry(leaf) == [] + + +def test_a_loop_in_the_parent_column_does_not_hang_the_backfill(): + # The column points at its own table, so bad data can make a cycle. + # Without the guard the migration never ends, and the deploy that + # runs it never finishes. + from catalog.models import ServiceComponent + + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service, parent=first) + # `update` skips `clean`, which is the only thing refusing this. + ServiceComponent.objects.filter(pk=first.pk).update(parent=second) + + backfill() + + assert ancestry(first) == [second.id] + assert ancestry(second) == [first.id] From 42521f654a4e4ca1c1dc531fa10761124f772406 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:58:05 -0700 Subject: [PATCH 49/94] fix: a component's path stops at its service `ancestors` walked `parent` without checking whose component it is. The admin allows moving a component to another service, which leaves its old children pointing across. The closure table gives such a row no ancestors, so `?ancestor=` and `descendant_count` left it out while `path` still drew a breadcrumb into the other service. One response held both answers. The rule now has one statement, on `ancestors`. The two other walks name it rather than repeat it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/models.py | 17 ++++++++++++++--- api/polling/reconcile.py | 11 ++++------- api/tests/test_catalog_models.py | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/api/catalog/models.py b/api/catalog/models.py index 1c5f16d..a28593f 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -516,11 +516,22 @@ def ancestors(self): """The components above this one, top down. A provider can nest a component under another. The chain is what - tells you where it sits on the status page. The guard is for bad - data: the column points at its own table, so a loop is possible. + tells you where it sits on the status page. + + Two rules end the walk. A loop is possible, because the column + points at its own table. And the chain stops at the service. + Moving a component leaves its old children pointing across. A + step there names a row this service's lists never hold. + + `polling.reconcile._chains` walks a whole service by the same + rules, and the backfill in migration 0008 repeats them. """ chain, node, seen = [], self.parent, {self.pk} - while node is not None and node.pk not in seen: + while ( + node is not None + and node.pk not in seen + and node.service_id == self.service_id + ): seen.add(node.pk) chain.append(node) node = node.parent diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index 3b93c7b..d48fd6c 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -101,13 +101,10 @@ def _chains(rows): parents = {row.id: row.parent_id for row in rows} def chain(node): - # The column points at its own table, so the data allows a loop. - # `seen` ends the walk. Without it bad data hangs the worker. - # - # `up in parents` keeps the chain inside this service. Moving - # a component to another service leaves its old children - # pointing across. A step there names a row this service's - # lists never hold. + # `seen` and `up in parents` are the loop guard and the service + # boundary. `ServiceComponent.ancestors` says why each ends the + # walk. The rows in hand are one service, so membership is the + # boundary here. walked, seen, up = [], {node}, parents.get(node) while up is not None and up not in seen and up in parents: seen.add(up) diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index e79a925..eaa6aa5 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -197,6 +197,23 @@ def test_a_move_between_services_rebuilds_the_service_left_behind(db): assert list(ServiceComponent.objects.search("programmable sms")) == [] +def test_a_parent_in_another_service_is_not_a_step_in_the_path(db): + # The closure table stops at the service boundary. A breadcrumb that + # walked past it names a component of another service. The same + # response then says the row has no ancestors. + old = ServiceFactory(name="Twilio") + top = ComponentFactory(service=old, name="Programmable Messaging") + leaf = ComponentFactory(service=old, name="SMS", parent=top) + + top.service = ServiceFactory(name="Vonage") + top.save() + + leaf.refresh_from_db() + assert leaf.ancestors == [] + assert leaf.visible_ancestors == [] + assert leaf.path == "Twilio / SMS" + + def test_a_save_that_moves_nothing_rebuilds_nothing(db, monkeypatch): # A rebuild rewrites every row of the service. A save of an # unrelated column must not pay for it, or one poll goes quadratic. From 9ce7c3ad0101255ed193f11c8cdb7a58d2be9e57 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 13:58:59 -0700 Subject: [PATCH 50/94] fix: a related count counts each row once `related_count` counted the rows a join returned. The tracked component count hands it a queryset joined through boards, so a component on two of one person's boards counted twice. A client reads that number as how many of a service's components sit on the board, and the tab below it listed one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/common/queries.py | 6 +++++- api/tests/test_catalog_models.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/api/common/queries.py b/api/common/queries.py index 59ab896..ddc88a1 100644 --- a/api/common/queries.py +++ b/api/common/queries.py @@ -14,12 +14,16 @@ def related_count(queryset, group_by, ref="pk"): Several counts on one row multiply into one another. Each is a join, so a service is read once per component per event. Each count asks its own question instead. + + Distinct, because the caller may hand in a queryset that already + joins. A component on two of one person's boards is one component, + and it arrived twice. """ counted = ( queryset.filter(**{group_by: OuterRef(ref)}) .order_by() .values(group_by) - .annotate(total=Count("pk")) + .annotate(total=Count("pk", distinct=True)) .values("total") ) return Coalesce(Subquery(counted, output_field=IntegerField()), 0) diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index eaa6aa5..b8d7424 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -370,6 +370,23 @@ def test_a_services_tracked_count_leaves_out_an_archived_row(db): assert prepared.tracked_component_count == 0 +def test_a_services_tracked_count_counts_a_component_once_per_owner(db): + # A person may own several boards. The count says how many of this + # service's components are on them. Counting the join rows makes one + # component held twice read as two, above a tab listing one. + from dashboards.models import Dashboard, DashboardItem + + user = UserFactory() + service = ServiceFactory() + component = ComponentFactory(service=service) + track(component, user=user) + second = Dashboard.objects.create(owner=user) + DashboardItem.objects.create(dashboard=second, component=component) + + prepared = Service.objects.for_display(user).get(pk=service.pk) + assert prepared.tracked_component_count == 1 + + def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): # The rollup's own name is an exact, weight-A hit. A leaf matches # only through its service, at weight C, so it must sort below. From ad1953a75973f46d4444a9da32fe6b7c427aadf3 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 14:00:12 -0700 Subject: [PATCH 51/94] fix: the audit log drops the search vector `search_document` is derived, and `rebuild_search` writes it in a bulk update that records no history. Every historical row therefore held the vector of the last ordinary save. An audit log answers who changed what. A column no editor sets, and whose recorded value is wrong, answers nothing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- .../0009_history_drops_search_document.py | 16 ++++++++++++++++ api/catalog/models.py | 5 ++++- api/tests/test_catalog_models.py | 8 ++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 api/catalog/migrations/0009_history_drops_search_document.py diff --git a/api/catalog/migrations/0009_history_drops_search_document.py b/api/catalog/migrations/0009_history_drops_search_document.py new file mode 100644 index 0000000..f280627 --- /dev/null +++ b/api/catalog/migrations/0009_history_drops_search_document.py @@ -0,0 +1,16 @@ +# Generated by Django 5.2.17 on 2026-09-04 20:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0008_component_ancestor_table"), + ] + + operations = [ + migrations.RemoveField( + model_name="historicalservicecomponent", + name="search_document", + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index a28593f..9368dbe 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -355,7 +355,10 @@ class ServiceComponent(BaseModel): verbose_name="Archived on", null=True, blank=True, editable=False ) - history = HistoricalRecords() + # `search_document` is derived and `rebuild_search` writes it in a + # bulk update, which records no history. A kept column would hold + # the vector of the last ordinary save on every old row. + history = HistoricalRecords(excluded_fields=["search_document"]) class Meta(BaseModel.Meta): constraints = [ diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index b8d7424..55cff62 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -387,6 +387,14 @@ def test_a_services_tracked_count_counts_a_component_once_per_owner(db): assert prepared.tracked_component_count == 1 +def test_the_audit_log_holds_no_search_vector(): + # `rebuild_search` writes the column in a bulk update, which records + # no history. Every old row would carry the vector of the last + # ordinary save. That describes neither the row nor the edit. + columns = {field.name for field in ServiceComponent.history.model._meta.fields} + assert "search_document" not in columns + + def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): # The rollup's own name is an exact, weight-A hit. A leaf matches # only through its service, at weight C, so it must sort below. From d872be5547f2f12e570fe1208fc86b49ee0f6be5 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 14:02:19 -0700 Subject: [PATCH 52/94] docs: the layout rules describe the code as it is Two lines were narrower than the codebase. `common` was said never to import an app, and nine imports do. The rule those imports keep is that `common` holds no expression describing an app. Publishing an app's choices is what `/meta/` is for, and one place has to list them. `catalog` was said to hold one module per resource, and it holds four modules for three resources. `imports.py` is an operation on `Service`. A rules file the code breaks reads as aspirational, or sends the next reader to fix what is right. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0bc2fc6..4f0e123 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,9 +62,10 @@ problem; the setting fixes the class. Which file a piece of code goes in, by what it is. - An app keeps one `views.py` until it serves more than one resource. - Then it becomes `views/`, one module per resource. `catalog` serves - three. A sub-resource stays with its parent, so an event's update - log is not a second module. + Then it becomes `views/`, one module per resource or per long-lived + operation. `catalog` serves three resources, and `imports.py` is an + operation on `Service`. A sub-resource stays with its parent, so an + event's update log is not a second module. - A **queryset method** is chainable vocabulary. It lives on the queryset in `models.py`, because a caller chains it onto a query. - A **bare expression** is handed to `annotate()`. It lives in @@ -72,9 +73,11 @@ Which file a piece of code goes in, by what it is. - `queries.py` is row-scoped and `aggregates.py` is collection-scoped. One lands in a serialized row. The other lands in the `aggregates` key beside `results`. -- `common` never imports an app. `common/ordering.py` records this as a - scar. Those expressions lived there, and the base layer imported the - app it describes. +- `common` holds no expression that describes an app. `common/ordering.py` + records this as a scar. The domain subqueries lived there, and the base + layer imported the app it describes. Naming an app's choices to publish + them is another thing, and it is fine: `/meta/` lists every enum, and + one place has to. ## Settings From 67445babff4af2ffcdcabac2fd5b4eced0c2a06b Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 14:02:20 -0700 Subject: [PATCH 53/94] docs: the rollup's zero descendant count is counted, not promised The serializer reads the annotation. The method that forced 0 on the overall component is gone, and the admin's parent picker still allows a component under the rollup. `PathNode.is_overall` exists because the rollup can be a step in a path. No adapter nests anything under it, so the number is 0 in the data. The contract now says that, rather than a guarantee nothing holds. `path` is also empty on the rollup, not null. The contract said both, fifty lines apart. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- docs/api/openapi.yaml | 11 ++++++----- docs/specs/2026-08-23-statusboard-design.md | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index b4e4d26..b179a87 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -827,9 +827,9 @@ components: rather than a copy of its fields. One FK, one nested serializer. It is a plain `Component`, the same shape a component has anywhere else, so one - renderer draws every row. Five of its fields are fixed by what an overall component is: - `parent` and `path` are null, `descendant_count` is 0, `is_overall` is true, and `name` - is the service name. They are returned rather than omitted. A client that + renderer draws every row. Five of its fields follow from what an overall component is: + `parent` is null, `path` is empty, `descendant_count` is 0, `is_overall` is true, and + `name` is the service name. They are returned rather than omitted. A client that special-cased them would need a second renderer to gain nothing. This is the one place an archived component is served. A provider can stop publishing @@ -888,8 +888,9 @@ components: How many components sit anywhere under this one. A component is a group when this is above zero. There is no is_group flag. - Always 0 on the overall component. It is a peer of the top-level components, not their - parent. + 0 on the overall component. The rollup is a peer of the top-level components, not + their parent, and no adapter nests anything under it. The number is counted rather + than fixed, so a component parented under the rollup would raise it. is_overall: { type: boolean, description: The component Statusboard creates per service, carrying the provider's top-level indicator. Always first, parent is null. } archived_at: { type: string, format: date-time, nullable: true, description: 'When the provider stopped publishing the component. An archived component is not served, so this is null everywhere but `Service.overall_component`.' } status: { $ref: '#/components/schemas/Status' } diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index ec01cdd..68aa828 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -409,8 +409,8 @@ Unique on `(dashboard, component)`. ### Every service has an overall component Statusboard creates one component per service, `is_overall = true`, `parent = null`, named -"All services". `path` is null, since it has no ancestry, and `descendant_count` is 0 — it is a peer of -the top-level components rather than their parent. +"All services". `path` is empty, since it has no ancestry, and `descendant_count` is 0. It is a peer of +the top-level components rather than their parent, and no adapter nests anything under it. **`component_count` excludes it.** The number equals what the provider's own status page claims, which is the number a user can check. The components list returns the overall row first and does From d2a09e0e34dc50d3be9c7fb5c2efb64e3013a0f5 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 14:02:20 -0700 Subject: [PATCH 54/94] docs: two test comments describe the suite as it stands One comment explained `update_fields` by a column this branch deleted. The call is still right, so only the reason goes. The archived guard derives its paths from the contract. It catches a new operation, and it cannot catch a new reader on a path it already covers. A reader who does not know that trusts it too far. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_archived.py | 6 ++++++ api/tests/test_catalog_models.py | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/api/tests/test_archived.py b/api/tests/test_archived.py index 2fe14f7..00f489c 100644 --- a/api/tests/test_archived.py +++ b/api/tests/test_archived.py @@ -9,6 +9,12 @@ `tracked_component_count` and the board write. Each was caught by hand in review. Remembering has failed three times, so the suite asks instead. + +The last test derives its list of paths from the contract. So it fails +on an operation that serves a component and has no probe. It says +nothing about a new reader on a path a probe already covers. Another +query parameter, or a nested component on a schema that path returns, +adds no path and trips nothing. """ from pathlib import Path diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 55cff62..24838f6 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -322,8 +322,6 @@ def test_the_descendant_count_leaves_out_an_archived_row(db): middle = ComponentFactory(service=service, parent=top) leaf = ComponentFactory(service=service, parent=middle) rebuild_ancestry(service) - # `update_fields`, because this row is stale: `rebuild_ancestry` - # wrote its ancestry after the factory made it. leaf.is_archived = True leaf.save(update_fields=["is_archived"]) From 5d2f9464e7bbb41e23853882d6b07b8cbfe10b15 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 14:33:57 -0700 Subject: [PATCH 55/94] fix: a database trigger refuses a component parented under the rollup ServiceComponent.descendant_count() used to force 0 on the overall component, and that guard also kept it from ever being a parent. The method was deleted with the annotation fallbacks, and nothing replaced the guard: the admin's parent picker still offered the rollup, and nothing in the schema refused the row. A component parented under the rollup corrupts every descendant count and every breadcrumb on that service, so this is an invariant the code depends on, not a cosmetic default. A plain CheckConstraint cannot express it: is_overall belongs to the parent row, not the row being checked, and Postgres refuses a joined condition in a CHECK. A trigger can, because it runs a query instead of evaluating an expression. It checks both directions: a row parented under the rollup, and a row with children gaining the flag, since is_overall is editable after the fact. clean() gains the same check, so a form sees a readable error instead of the trigger's IntegrityError. The admin's parent queryset now excludes the rollup unconditionally, on add as well as change, so the picker stops offering the one choice the database refuses. Three existing tests parented a component under the rollup incidentally, to give it an ancestor; they now use an ordinary component for that instead, and the contract's descendant_count sentence is restored to a guarantee now that the database backs it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 11 +++- .../migrations/0010_rollup_is_not_a_parent.py | 58 +++++++++++++++++++ api/catalog/models.py | 18 +++++- api/tests/test_admin.py | 24 ++++++++ api/tests/test_catalog_api.py | 28 ++++----- api/tests/test_catalog_models.py | 33 +++++++++++ api/tests/test_cross_service.py | 15 +++++ docs/api/openapi.yaml | 6 +- docs/specs/2026-08-23-statusboard-design.md | 5 +- 9 files changed, 174 insertions(+), 24 deletions(-) create mode 100644 api/catalog/migrations/0010_rollup_is_not_a_parent.py diff --git a/api/catalog/admin.py b/api/catalog/admin.py index 7547220..6766662 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -590,12 +590,17 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): self.admin_site, scope={"service": editing.service_id if editing else None}, ) + # The rollup is never a parent. Excluding it here, on both + # add and change, keeps the picker from offering the one + # choice the database refuses. + queryset = ServiceComponent.objects.exclude(is_overall=True) if editing is not None: # The server's half: the picker narrows what is offered, # this refuses anything else that is posted. - kwargs["queryset"] = ServiceComponent.objects.filter( - service_id=editing.service_id - ).exclude(pk=editing.pk) + queryset = queryset.filter(service_id=editing.service_id).exclude( + pk=editing.pk + ) + kwargs["queryset"] = queryset return super().formfield_for_foreignkey(db_field, request, **kwargs) def delete_queryset(self, request, queryset): diff --git a/api/catalog/migrations/0010_rollup_is_not_a_parent.py b/api/catalog/migrations/0010_rollup_is_not_a_parent.py new file mode 100644 index 0000000..b79bedb --- /dev/null +++ b/api/catalog/migrations/0010_rollup_is_not_a_parent.py @@ -0,0 +1,58 @@ +from django.db import migrations + +# `is_overall` belongs to the parent row, not the row being written. A +# `CheckConstraint` sees one row, so it cannot read a joined column and +# cannot express "my parent is not the rollup". A trigger can, because +# it runs a query rather than evaluating an expression. +# +# The function checks both directions of the same rule. A row cannot be +# inserted or reparented under the rollup. A row cannot gain the rollup +# flag while something already points at it as a parent. Either gap +# would corrupt every descendant count and breadcrumb under it. +CREATE_TRIGGER = """ +CREATE FUNCTION catalog_component_rollup_is_not_a_parent() +RETURNS trigger AS $$ +BEGIN + IF NEW.parent_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM catalog_servicecomponent + WHERE id = NEW.parent_id AND is_overall + ) THEN + RAISE EXCEPTION + 'rollup_is_not_a_parent: % cannot be parented under the overall component', + NEW.id + USING ERRCODE = 'check_violation'; + END IF; + + IF NEW.is_overall AND EXISTS ( + SELECT 1 FROM catalog_servicecomponent WHERE parent_id = NEW.id + ) THEN + RAISE EXCEPTION + 'rollup_is_not_a_parent: % already has children, so it cannot become the overall component', + NEW.id + USING ERRCODE = 'check_violation'; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER rollup_is_not_a_parent + BEFORE INSERT OR UPDATE OF parent_id, is_overall ON catalog_servicecomponent + FOR EACH ROW + EXECUTE FUNCTION catalog_component_rollup_is_not_a_parent(); +""" + +DROP_TRIGGER = """ +DROP TRIGGER IF EXISTS rollup_is_not_a_parent ON catalog_servicecomponent; +DROP FUNCTION IF EXISTS catalog_component_rollup_is_not_a_parent(); +""" + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0009_history_drops_search_document"), + ] + + operations = [ + migrations.RunSQL(sql=CREATE_TRIGGER, reverse_sql=DROP_TRIGGER), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index 9368dbe..3e96f92 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -360,6 +360,12 @@ class ServiceComponent(BaseModel): # the vector of the last ordinary save on every old row. history = HistoricalRecords(excluded_fields=["search_document"]) + # The rollup is never a parent, but `is_overall` belongs to the + # parent row, not this one. A `CheckConstraint` sees one row, so it + # cannot read a joined column and cannot express this. Migration + # 0010 adds a trigger instead, named `rollup_is_not_a_parent`. It + # checks both directions: a row parented under the rollup, and a + # row with children gaining the flag. class Meta(BaseModel.Meta): constraints = [ models.UniqueConstraint( @@ -500,9 +506,11 @@ def _services_to_rebuild(self, update_fields): def clean(self): """A component sits under one of its own service's components. - Nothing in the column says whose component the parent is. - Without this a service's tree reaches into another service's, - and a page shows one product's components under another. + Nothing in the column says whose component the parent is, or + that the parent is not the rollup. Without the first check a + service's tree reaches into another service's. Without the + second a rollup gains children, and every descendant count and + breadcrumb under it reads wrong. """ super().clean() if self.parent_id is None: @@ -513,6 +521,10 @@ def clean(self): raise ValidationError( {"parent": "That component belongs to another service."} ) + if self.parent.is_overall: + raise ValidationError( + {"parent": "The overall component is never a parent."} + ) @property def ancestors(self): diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index dea5151..87d09bf 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -934,6 +934,30 @@ def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): assert list(ServiceComponent.objects.search("programmable sms")) == [child] +@pytest.mark.django_db +def test_the_parent_picker_refuses_the_rollup(staff_client): + """The queryset, not just `clean`, is what a posted form meets first. + + A `ModelChoiceField` rejects a posted pk outside its queryset before + the instance is ever built, so `clean` never runs. Narrowing the + queryset is what turns a stray click into a redisplayed form + instead of a trigger's `IntegrityError`. + """ + from tests.factories import ComponentFactory, ServiceFactory + + service = ServiceFactory(name="Twilio") + rollup = ComponentFactory(service=service, is_overall=True) + child = ComponentFactory(service=service, name="SMS") + + url, data = component_form_data(staff_client, child, parent=str(rollup.pk)) + response = staff_client.post(url, data) + + assert response.status_code == 200 + assert b"valid choice" in response.content + child.refresh_from_db() + assert child.parent_id is None + + def bulk_delete(admin_client, components): """Run the admin's own delete action, past the confirmation page. diff --git a/api/tests/test_catalog_api.py b/api/tests/test_catalog_api.py index 8e5f0dd..eae9d4e 100644 --- a/api/tests/test_catalog_api.py +++ b/api/tests/test_catalog_api.py @@ -314,17 +314,17 @@ def test_a_path_carries_enough_to_link_to_each_step(): service = ServiceFactory(name="Twilio") overall = ComponentFactory(service=service, name="Twilio", is_overall=True) - parent = ComponentFactory( - service=service, name="Programmable Messaging", parent=overall - ) - ComponentFactory(service=service, name="SMS", parent=parent) + group = ComponentFactory(service=service, name="Programmable Messaging") + sms = ComponentFactory(service=service, name="SMS", parent=group) + ComponentFactory(service=service, name="Delivery", parent=sms) prepared = ServiceComponent.objects.for_display() - path = ComponentSerializer(prepared.get(name="SMS")).data["path"] + path = ComponentSerializer(prepared.get(name="Delivery")).data["path"] - assert [step["name"] for step in path] == ["Twilio", "Programmable Messaging"] - assert path[0]["id"] == str(overall.id) - assert path[0]["is_overall"] is True + assert [step["name"] for step in path] == ["Programmable Messaging", "SMS"] + assert path[0]["id"] == str(group.id) + # The rollup is never a parent, so a real path step never carries it. + assert path[0]["is_overall"] is False # Empty, not null: the overall component is under nothing, and a # client maps over the list without checking first. assert ComponentSerializer(prepared.get(pk=overall.pk)).data["path"] == [] @@ -336,13 +336,14 @@ def test_fields_prunes_inside_a_path_step(): # serializer. A dotted path into it was ignored, and the contract # promises a dotted path prunes wherever it is accepted. service = ServiceFactory(name="Twilio") - overall = ComponentFactory(service=service, name="Twilio", is_overall=True) - child = ComponentFactory(service=service, name="SMS", parent=overall) + ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + child = ComponentFactory(service=service, name="SMS", parent=parent) url = reverse("component-detail", args=[child.id]) body = APIClient().get(url, {"fields": "path.name"}).json() - assert body["path"] == [{"name": "Twilio"}] + assert body["path"] == [{"name": "Programmable Messaging"}] @pytest.mark.django_db @@ -350,8 +351,9 @@ def test_fields_refuses_an_unknown_name_under_a_path_step(): # Nothing pruned and nothing refused. A typo read as a step that # holds every field, which is the opposite of what was asked for. service = ServiceFactory(name="Twilio") - overall = ComponentFactory(service=service, name="Twilio", is_overall=True) - child = ComponentFactory(service=service, name="SMS", parent=overall) + ComponentFactory(service=service, name="Twilio", is_overall=True) + parent = ComponentFactory(service=service, name="Programmable Messaging") + child = ComponentFactory(service=service, name="SMS", parent=parent) url = reverse("component-detail", args=[child.id]) response = APIClient().get(url, {"fields": "path.nonsense"}) diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 24838f6..587e34a 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -161,6 +161,39 @@ def test_the_flag_and_the_date_cannot_disagree(): ServiceComponent.objects.filter(pk=component.pk).update(is_archived=True) +@pytest.mark.django_db +def test_the_database_refuses_a_component_parented_under_the_rollup(): + """A bulk write skips `clean`. Only the trigger is left to refuse it. + + `ComponentFactory` calls `objects.create`, never `full_clean`. A + poll or a shell session meets this, never a form's `clean` error. + """ + service = ServiceFactory() + rollup = ComponentFactory(service=service, is_overall=True) + with pytest.raises(IntegrityError): + ComponentFactory(service=service, parent=rollup) + + +@pytest.mark.django_db +def test_the_database_refuses_a_rollup_that_already_has_children(): + """The same rule, met from the other side. + + A component can gain the rollup flag after it already has + children, since `is_overall` is editable. The trigger checks both + directions, or promoting an existing parent would corrupt every + descendant count under it. + """ + from django.db import IntegrityError, transaction + + service = ServiceFactory() + parent = ComponentFactory(service=service) + ComponentFactory(service=service, parent=parent) + + parent.is_overall = True + with pytest.raises(IntegrityError), transaction.atomic(): + parent.save() + + def test_a_reparent_in_plain_python_rebuilds_the_derived_columns(db): # An admin-only hook left every other writer stale: a shell session, # a data migration, a later API write. The symptom is invisible. The diff --git a/api/tests/test_cross_service.py b/api/tests/test_cross_service.py index 634f0d0..08832e8 100644 --- a/api/tests/test_cross_service.py +++ b/api/tests/test_cross_service.py @@ -69,6 +69,21 @@ def test_a_component_may_sit_under_its_own_services_component(two_services): child.full_clean() +@pytest.mark.django_db +def test_a_component_cannot_sit_under_the_rollup(two_services): + # The rollup is a peer of the top-level components, not their + # parent. A database trigger refuses the row; this is what a form + # sees instead of that trigger's error. + mine, _ = two_services + rollup = ComponentFactory(service=mine, is_overall=True) + child = ComponentFactory(service=mine, external_id="child") + child.parent = rollup + + with pytest.raises(ValidationError) as raised: + child.full_clean() + assert "parent" in raised.value.message_dict + + @pytest.mark.django_db def test_a_reading_cannot_cite_a_poll_of_another_service(two_services): mine, theirs = two_services diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index b179a87..45529ce 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -888,9 +888,9 @@ components: How many components sit anywhere under this one. A component is a group when this is above zero. There is no is_group flag. - 0 on the overall component. The rollup is a peer of the top-level components, not - their parent, and no adapter nests anything under it. The number is counted rather - than fixed, so a component parented under the rollup would raise it. + Always 0 on the overall component. It is a peer of the top-level components, not + their parent. A database trigger refuses a component parented under it. The count + is a guarantee, not just what the data happens to hold. is_overall: { type: boolean, description: The component Statusboard creates per service, carrying the provider's top-level indicator. Always first, parent is null. } archived_at: { type: string, format: date-time, nullable: true, description: 'When the provider stopped publishing the component. An archived component is not served, so this is null everywhere but `Service.overall_component`.' } status: { $ref: '#/components/schemas/Status' } diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 68aa828..4ba12c4 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -409,8 +409,9 @@ Unique on `(dashboard, component)`. ### Every service has an overall component Statusboard creates one component per service, `is_overall = true`, `parent = null`, named -"All services". `path` is empty, since it has no ancestry, and `descendant_count` is 0. It is a peer of -the top-level components rather than their parent, and no adapter nests anything under it. +"All services". `path` is empty, since it has no ancestry. `descendant_count` is always 0. It is +a peer of the top-level components rather than their parent. A database trigger refuses a +component parented under it. **`component_count` excludes it.** The number equals what the provider's own status page claims, which is the number a user can check. The components list returns the overall row first and does From 50c37fb61531a35b1e0de3cbbe975311c6657727 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 14:59:46 -0700 Subject: [PATCH 56/94] refactor: a query answers ancestry and search, no stored copy `ComponentAncestor` and `search_document` each held a copy of what `parent` and `name` already say. A copy has to be rewritten every time the tree moves, so both needed a writer on `save`, on `delete`, in the admin's bulk delete, and a `held_rebuilds()` suppressor for polls. Miss one path and a component is silently absent from `?ancestor=`, from `?q=` and from every count. An untracked service is never polled, so nothing repairs it. Speed did not pay for that. A recursive CTE is one indexed query, so the walk down `parent` costs a fifth of a millisecond more on 341 components. A weighted `tsvector` beats a per-word `icontains` by about 1.5 ms on 4,800. Neither buys back four writers that can leave the answer wrong. The walk carries the ids it has visited, because `parent` is a self-FK and nothing forbids a loop. Each step tests `service_id`, because a component's tree stops at its own service. A count leaves archived rows out and still walks through them, so an archived group cannot hide the live rows under it. Search gives up stemming, relevance ranking, and reaching a leaf through the name of a group above it. The rollup leads a search instead, so a popular service cannot flood the list with its parts. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 20 -- api/catalog/filters.py | 12 +- .../0011_drop_derived_ancestry_and_search.py | 35 +++ api/catalog/models.py | 225 ++---------------- api/catalog/queries.py | 92 ++++++- api/catalog/views/components.py | 12 +- api/polling/reconcile.py | 145 +++-------- api/tests/factories.py | 21 +- api/tests/test_admin.py | 71 +----- api/tests/test_archived.py | 4 - api/tests/test_catalog_models.py | 220 ++++++----------- api/tests/test_component_api.py | 40 +--- api/tests/test_migrations.py | 96 -------- api/tests/test_reconcile.py | 93 +------- docs/api/openapi.yaml | 8 +- docs/check_docs.py | 2 +- 16 files changed, 297 insertions(+), 799 deletions(-) create mode 100644 api/catalog/migrations/0011_drop_derived_ancestry_and_search.py delete mode 100644 api/tests/test_migrations.py diff --git a/api/catalog/admin.py b/api/catalog/admin.py index 6766662..d0d31e4 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -40,7 +40,6 @@ from common.queries import related_count from polling.importer import import_from_url from polling.models import Poller, PollRun -from polling.reconcile import rebuild_ancestry, rebuild_search from status.choices import EventKind, Severity from status.models import ComponentStatus, ServiceEvent from status.queries import CURRENT_SEVERITY @@ -603,25 +602,6 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): kwargs["queryset"] = queryset return super().formfield_for_foreignkey(db_field, request, **kwargs) - def delete_queryset(self, request, queryset): - """Rebuild each service the selection touches, once. - - `delete_selected` is how an admin removes several components, - and it never reaches `ServiceComponent.delete`. Without this a - deleted middle row leaves its parent linked to its children. - `parent` is SET_NULL, so the orphan reads as a root. - - Once per service, not once per row. A rebuild rewrites the whole - tree, so a selection of fifty would otherwise cost fifty passes. - - Read the services before the delete. The rows are gone after it. - """ - services = list(Service.objects.filter(pk__in=queryset.values("service_id"))) - super().delete_queryset(request, queryset) - for service in services: - rebuild_ancestry(service) - rebuild_search(service) - def get_queryset(self, request): return ( super() diff --git a/api/catalog/filters.py b/api/catalog/filters.py index 5e35c6a..6f4a480 100644 --- a/api/catalog/filters.py +++ b/api/catalog/filters.py @@ -8,6 +8,7 @@ from django_filters import rest_framework as filters from catalog.models import ServiceComponent +from catalog.queries import descendant_ids from common.filters import SeverityFilterMixin @@ -15,7 +16,7 @@ class ComponentFilter(SeverityFilterMixin): service = filters.CharFilter(field_name="service__slug") # Every descendant, not one level. `parent` would name a query this # does not run. - ancestor = filters.UUIDFilter(field_name="ancestor_links__ancestor") + ancestor = filters.UUIDFilter(method="filter_ancestor") event = filters.UUIDFilter(field_name="events__id") # `for_display` annotates this per viewer. It is not a column, so # there is nothing to generate the filter from. @@ -25,6 +26,15 @@ class Meta: model = ServiceComponent fields = {"is_overall": ["exact"]} + def filter_ancestor(self, queryset, name, value): + """Ask the tree for the ids below, then narrow to them. + + `catalog.queries.descendant_ids` is the one walk down `parent`. + A join written here would be a second answer to the question a + descendant count already asks. + """ + return queryset.filter(pk__in=descendant_ids(value)) + def filter_is_tracked(self, queryset, name, value): """Answer it here, because nobody signed out tracks anything. diff --git a/api/catalog/migrations/0011_drop_derived_ancestry_and_search.py b/api/catalog/migrations/0011_drop_derived_ancestry_and_search.py new file mode 100644 index 0000000..8a985f8 --- /dev/null +++ b/api/catalog/migrations/0011_drop_derived_ancestry_and_search.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.17 on 2026-09-04 21:48 + +from django.db import migrations + +# Ancestry and the search vector were copies of what `parent` and `name` +# already say. A copy goes stale, so each one needed a writer on every +# path that could move a component. A query answers both instead. + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0010_rollup_is_not_a_parent"), + ] + + operations = [ + migrations.RemoveField( + model_name="componentancestor", + name="ancestor", + ), + migrations.RemoveField( + model_name="componentancestor", + name="descendant", + ), + migrations.RemoveIndex( + model_name="servicecomponent", + name="components_by_search", + ), + migrations.RemoveField( + model_name="servicecomponent", + name="search_document", + ), + migrations.DeleteModel( + name="ComponentAncestor", + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index 3e96f92..89ae060 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -1,10 +1,5 @@ -import uuid -from contextlib import contextmanager -from contextvars import ContextVar from urllib.parse import urlparse, urlunparse -from django.contrib.postgres.indexes import GinIndex -from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVectorField from django.core.exceptions import ValidationError from django.db import models from django.utils import timezone @@ -211,30 +206,6 @@ def __str__(self): return self.url -_rebuilds_held = ContextVar("catalog_rebuilds_held", default=False) - - -@contextmanager -def held_rebuilds(): - """Hold back the per-save rebuild, for a caller doing its own. - - `ServiceComponent.save` rewrites the whole service. A caller that - saves many components of one service pays that per row. A poll of a - hundred components then costs a hundred passes over a hundred rows. - - A caller that holds the rebuild owes the service one afterwards. - `polling.reconcile._upsert_components` is the only one. - - A `ContextVar` rather than a flag, so one worker thread cannot hold - back another's rebuild. - """ - token = _rebuilds_held.set(True) - try: - yield - finally: - _rebuilds_held.reset(token) - - class ServiceComponentQuerySet(models.QuerySet): def visible(self): """The components a caller is served, anywhere. @@ -250,17 +221,22 @@ def visible(self): return self.filter(is_archived=False) def search(self, q): - """Rank a query against the stored path. + """Narrow to what every typed word describes. - Websearch mode, so several words are an AND. A caller ordering - by anything else drops the rank and keeps the filter. + A word matches the component's own name or its service's, so + `twilio` reaches Twilio's SMS row. Several words are an AND, so + `twilio sms` reaches that row and not its siblings. + + The result is unordered. The list view decides what leads a + search. An ordering backend runs after this and would discard + any order set here. """ - query = SearchQuery(q, search_type="websearch") - return ( - self.filter(search_document=query) - .annotate(rank=SearchRank(models.F("search_document"), query)) - .order_by("-rank") - ) + rows = self + for term in q.split(): + rows = rows.filter( + models.Q(name__icontains=term) | models.Q(service__name__icontains=term) + ) + return rows def for_display(self, user=None): """Everything `ComponentSerializer` reads, in four queries. @@ -328,10 +304,6 @@ class ServiceComponent(BaseModel): on_delete=models.SET_NULL, related_name="children", ) - # The whole path, weighted, written by reconcile. Searching a - # service's name must also reach its components. Walking `parent` - # at query time cannot use an index. - search_document = SearchVectorField(null=True, editable=False) objects = ServiceComponentQuerySet.as_manager() status_page_order = models.IntegerField(verbose_name="Page order", default=0) @@ -355,10 +327,7 @@ class ServiceComponent(BaseModel): verbose_name="Archived on", null=True, blank=True, editable=False ) - # `search_document` is derived and `rebuild_search` writes it in a - # bulk update, which records no history. A kept column would hold - # the vector of the last ordinary save on every old row. - history = HistoricalRecords(excluded_fields=["search_document"]) + history = HistoricalRecords() # The rollup is never a parent, but `is_overall` belongs to the # parent row, not this one. A `CheckConstraint` sees one row, so it @@ -382,9 +351,6 @@ class Meta(BaseModel.Meta): name="archived_flag_and_date_agree", ), ] - indexes = [ - GinIndex(fields=["search_document"], name="components_by_search"), - ] def live_events(self, kind): """The live events of one kind. @@ -414,24 +380,12 @@ def is_tracked_by(self, user): return self.boards.filter(owner=user).exists() return prepared - # The ancestry links and `search_document` are built from these. A - # save touching none of them leaves both true. - REBUILD_TRIGGERS = frozenset( - {"name", "parent", "parent_id", "service", "service_id"} - ) - def save(self, *args, **kwargs): - """Keep the archive date in step with the flag, and the tree true. + """Keep the archive date in step with the flag. A bulk update does not come through here, so anything writing the flag that way sets the date itself. The constraint is what makes that a failure rather than a quiet disagreement. - - The rebuild was on the admin alone. Every other writer left the - derived columns stale. The symptom is silence: the component is - absent from `?ancestor=`, from `?q=` and from every - `descendant_count`. A poll repairs a tracked service, so a - paused or untracked one stayed wrong forever. """ if self.is_archived and self.archived_at is None: self.archived_at = timezone.now() @@ -440,68 +394,7 @@ def save(self, *args, **kwargs): fields = kwargs.get("update_fields") if fields is not None and "is_archived" in fields: kwargs["update_fields"] = [*fields, "archived_at"] - stale = self._services_to_rebuild(kwargs.get("update_fields")) super().save(*args, **kwargs) - if stale: - from polling.reconcile import rebuild_ancestry, rebuild_search - - for service in stale: - rebuild_ancestry(service) - rebuild_search(service) - - def delete(self, *args, **kwargs): - """Rebuild the service, because a delete moves the rows below. - - The cascade drops this row's own links. `parent` is SET_NULL, - so a child of this row becomes a root. The link from this row's - parent down to that child survives. It names a tree nobody - sits in. - - A queryset delete does not come through here. Nothing removes a - component that way. A poll archives instead. - - Read the service before the write. Django clears the key. - """ - service = self.service - result = super().delete(*args, **kwargs) - from polling.reconcile import rebuild_ancestry, rebuild_search - - rebuild_ancestry(service) - rebuild_search(service) - return result - - def _services_to_rebuild(self, update_fields): - """Whose derived columns this save is about to invalidate. - - Empty is the normal answer. A poll saves every row of a service - and usually moves none of them. So the check costs no query when - nothing the columns are built from changed. - - A create counts. A component made in a shell otherwise holds no - ancestry and no search vector. - - Call this before the write. It reads the row as it still stands. - """ - if _rebuilds_held.get(): - return () - if self._state.adding: - return (self.service,) - if update_fields is not None and self.REBUILD_TRIGGERS.isdisjoint( - update_fields - ): - return () - was = ( - ServiceComponent.objects.filter(pk=self.pk) - .values_list("name", "parent_id", "service_id") - .first() - ) - if was is None or was == (self.name, self.parent_id, self.service_id): - return () - if was[2] == self.service_id: - return (self.service,) - # A move between services leaves the old one behind. Its - # ancestry links and its search documents still name this row. - return (self.service, Service.objects.get(pk=was[2])) def clean(self): """A component sits under one of its own service's components. @@ -538,8 +431,8 @@ def ancestors(self): Moving a component leaves its old children pointing across. A step there names a row this service's lists never hold. - `polling.reconcile._chains` walks a whole service by the same - rules, and the backfill in migration 0008 repeats them. + `catalog.queries` walks the other direction under the same two + rules. The rows here are already loaded, so this stays Python. """ chain, node, seen = [], self.parent, {self.pk} while ( @@ -574,85 +467,3 @@ def path(self): def __str__(self): return self.path - - -class ComponentAncestorQuerySet(models.QuerySet): - def to_visible(self): - """The links a caller is served, anywhere. - - `ServiceComponentQuerySet.visible` is the one definition of - which components a reader sees. Naming the archive flag here - would give a descendant count a second answer to that. - """ - return self.filter(descendant__in=ServiceComponent.objects.visible()) - - -class ComponentAncestor(models.Model): - """One component sits above another, at a known distance. - - Read the two names off the row, not off the component you hold. - They swap sides depending on where you stand. So a component's own - ancestors are `component.ancestor_links`, and everything under it is - `component.descendant_links`. - - `depth` counts the steps down from `ancestor` to `descendant`. A - parent is 1 and a grandparent is 2. A breadcrumb reads root first - with `order_by("-depth")`, and `depth=1` is the direct children. - - A component is not its own ancestor. Every reader counts other - components, so a depth 0 row would add one to every answer. - - This is not a `BaseModel`. Reconcile derives every row from - `parent`, so an author and an edit time record nothing anybody - reads. The key still follows the house rule. - - The pair replaces an array of ancestor ids. The array held no - reference. Deleting a component left its id in every descendant, - until the next poll of that service. An untracked service is never - polled again. - """ - - id = models.UUIDField(primary_key=True, default=uuid.uuid7, editable=False) - # The delete rule is the reason this table exists. A component that - # goes takes every claim about where it sat with it. - # - # `db_index=False` because the unique constraint already indexes - # this column first. That index answers the descendant direction: - # `?ancestor=`, and the count on every component row. - ancestor = models.ForeignKey( - ServiceComponent, - on_delete=models.CASCADE, - related_name="descendant_links", - db_index=False, - ) - # The foreign key index answers the ancestor direction: one - # component's own chain, which a breadcrumb reads. - descendant = models.ForeignKey( - ServiceComponent, - on_delete=models.CASCADE, - related_name="ancestor_links", - ) - depth = models.PositiveIntegerField() - objects = ComponentAncestorQuerySet.as_manager() - - class Meta: - # Root first, which is how a breadcrumb reads. - ordering = ["-depth"] - constraints = [ - models.UniqueConstraint( - fields=["ancestor", "descendant"], name="one_link_per_component_pair" - ), - # Both hold "no component is its own ancestor", which every - # count depends on. One forbids the row, one forbids the - # distance that would describe it. - models.CheckConstraint( - condition=~models.Q(ancestor=models.F("descendant")), - name="ancestry_excludes_self", - ), - models.CheckConstraint( - condition=models.Q(depth__gte=1), name="ancestry_depth_starts_at_one" - ), - ] - - def __str__(self): - return f"{self.ancestor} > {self.descendant} ({self.depth})" diff --git a/api/catalog/queries.py b/api/catalog/queries.py index d2cd2e8..e50e7ea 100644 --- a/api/catalog/queries.py +++ b/api/catalog/queries.py @@ -1,13 +1,18 @@ """What a caller reads about the catalog, as SQL. -Each one is a bare expression, handed to `annotate()`. Chainable +Most are a bare expression, handed to `annotate()`. Chainable vocabulary is a queryset method, and lives on the queryset instead. +The walk down `parent` is here too. It is raw SQL because a recursive +CTE has no ORM form, and it answers a catalog read. + These lived in `common.ordering`, which made the base layer import the app it describes. """ -from django.db.models import Count, Exists, OuterRef, Subquery +from django.db import connection +from django.db.models import Count, Exists, IntegerField, OuterRef, Subquery +from django.db.models.expressions import RawSQL from common.queries import related_count from status.models import ComponentStatus @@ -55,13 +60,84 @@ def component_count(): ) +# Everything under one component, walked down `parent`. `?ancestor=` +# and every descendant count ask this one question, so they read one +# definition of it. +# +# `walked` carries the ids already visited, and the step drops a row +# that is in it. `parent` points at this same table and nothing forbids +# a loop, so an unguarded walk never ends. A depth ceiling would end it +# too, but it would also cut a deep tree that is not a loop. +# +# The step tests `service_id` as well. A component's tree stops at its +# own service, which is the rule `ServiceComponent.ancestors` follows +# climbing the other way. +# +# `steps` is 0 on the component itself. No reader counts a component +# among the ones below it. +_BELOW = """ +WITH RECURSIVE below AS ( + SELECT seed.id, seed.is_archived, seed.service_id, + ARRAY[seed.id] AS walked, 0 AS steps + FROM {table} seed + WHERE seed.id = {root} + UNION ALL + SELECT step.id, step.is_archived, step.service_id, + below.walked || step.id, below.steps + 1 + FROM {table} step + JOIN below ON step.parent_id = below.id + AND step.service_id = below.service_id + WHERE NOT step.id = ANY(below.walked) +) +""" + + +def _table(): + """The component table, quoted. Read late, so no import runs early.""" + from catalog.models import ServiceComponent + + return connection.ops.quote_name(ServiceComponent._meta.db_table) + + +def _below(root, tail): + """The walk, rooted where the caller says, ending as it asks.""" + return _BELOW.format(table=_table(), root=root) + tail + + +class _CountsOwnRow(RawSQL): + """Raw SQL that reads the row it is annotated on. + + Django groups a page by every column it selects, and it cannot see + inside raw SQL. Left alone it groups by the whole subquery, which + Postgres refuses. The row's key is already grouped, and this counts + from that key alone. + """ + + def get_group_by_cols(self): + return [] + + +def descendant_ids(component_id): + """Every component under one, at any depth, as a subquery. + + `?ancestor=` reads this. An archived row is walked through and not + dropped, because the caller narrows the result to what it serves. + """ + return RawSQL(_below("%s", "SELECT id FROM below WHERE steps > 0"), [component_id]) + + def descendant_count(): """How many components sit under each row, in one query for the page. - Archived rows are left out. A caller is not served one anywhere - else, so a count that included them would name components no list - returns. + Archived rows are left out of the total. A caller is not served one + anywhere else, so a count that held them would name components no + list returns. The walk still passes through them, or an archived + group would hide the live rows under it. """ - from catalog.models import ComponentAncestor - - return related_count(ComponentAncestor.objects.to_visible(), "ancestor") + # The row being annotated, named by the table itself. Django never + # aliases the base table of a query, and the walk aliases its own. + sql = _below( + f"{_table()}.{connection.ops.quote_name('id')}", + "SELECT count(*) FROM below WHERE steps > 0 AND NOT is_archived", + ) + return _CountsOwnRow(f"({sql})", [], output_field=IntegerField()) diff --git a/api/catalog/views/components.py b/api/catalog/views/components.py index ef2382c..f98a3e4 100644 --- a/api/catalog/views/components.py +++ b/api/catalog/views/components.py @@ -64,19 +64,19 @@ def initial(self, request, *args, **kwargs): """ super().initial(request, *args, **kwargs) if request.query_params.get("q"): - # `search` already ranked the rows. A sort applied after it - # throws the ranking away. - self.ordering = ["-rank"] + # The rollup leads, then the suggested keys. Typing a + # service's name matches every part of it. Without this, + # eighty rows bury the one that answers the question. + self.ordering = ["-is_overall", *self.SUGGESTED] elif request.query_params.get("service"): # A service tab reads down the provider's own page. self.ordering = ["status_page_order"] def filter_queryset(self, queryset): - """Rank by relevance when the caller typed something. + """Narrow to what the caller typed, before the backends run. One control and one label, Smart, on every list. A separate - "best match" would name a ranking that does not exist until - somebody types. + "best match" would name a ranking the search does not work out. """ q = self.request.query_params.get("q") if q: diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index d48fd6c..c1c405d 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -1,10 +1,8 @@ from django.contrib.auth import get_user_model -from django.contrib.postgres.search import SearchVector from django.db import transaction -from django.db.models import Value from django.utils import timezone -from catalog.models import ComponentAncestor, ServiceComponent, held_rebuilds +from catalog.models import ServiceComponent from polling.system_events import claim, reconcile_system_events from status.models import ComponentStatus, EventUpdate, ServiceEvent @@ -56,123 +54,34 @@ def _signed(author, **fields): def _upsert_components(service, components, author): rows = {} - # `ServiceComponent.save` rebuilds the whole service. This pass - # saves every component of it, twice over, so the hook would make - # one poll quadratic. It runs once at the end instead. - with held_rebuilds(): - for incoming in components: - row, _ = ServiceComponent.objects.update_or_create( - service=service, - external_id=incoming.external_id, - **_signed( - author, - name=incoming.name, - status_page_order=incoming.order, - is_overall=incoming.is_overall, - is_archived=False, - archived_at=None, - ), - ) - rows[incoming.external_id] = row - - # Parents second: a child may arrive before its parent exists. - for incoming in components: - parent = ( - rows.get(incoming.parent_external_id) - if incoming.parent_external_id - else None - ) - row = rows[incoming.external_id] - if row.parent_id != (parent.id if parent else None): - row.parent = parent - row.updated_by = author - row.save(update_fields=["parent", "updated_by"]) - rebuild_ancestry(service) - rebuild_search(service) - return rows - - -def _chains(rows): - """Each row's ancestors, root first, keyed by id. - - The walk reads the rows in hand, not the database. A deep tree costs - no extra query, and both rebuilds ask the same question once. - """ - parents = {row.id: row.parent_id for row in rows} - - def chain(node): - # `seen` and `up in parents` are the loop guard and the service - # boundary. `ServiceComponent.ancestors` says why each ends the - # walk. The rows in hand are one service, so membership is the - # boundary here. - walked, seen, up = [], {node}, parents.get(node) - while up is not None and up not in seen and up in parents: - seen.add(up) - walked.append(up) - up = parents.get(up) - return list(reversed(walked)) - - return {row.id: chain(row.id) for row in rows} - - -def rebuild_ancestry(service): - """Write one link per pair of components, one above the other. - - A rename does not touch this. A reparent moves a whole subtree, so - the pass rewrites the service rather than one row. A service holds - hundreds of components, not millions. - - Only the rows that moved are written. A delete and a re-insert - would rewrite every link of the service on every poll. Almost no - poll moves a component. - """ - rows = list(ServiceComponent.objects.filter(service=service)) - # Root first, so the last step is the parent and depth counts down. - wanted = { - (up, row_id, len(chain) - step) - for row_id, chain in _chains(rows).items() - for step, up in enumerate(chain) - } - held = { - (link.ancestor_id, link.descendant_id, link.depth): link.pk - for link in ComponentAncestor.objects.filter(descendant__service=service) - } - stale = [pk for link, pk in held.items() if link not in wanted] - if stale: - # Before the insert. A pair whose depth changed holds the unique - # constraint until its old row goes. - ComponentAncestor.objects.filter(pk__in=stale).delete() - ComponentAncestor.objects.bulk_create( - [ - ComponentAncestor(ancestor_id=up, descendant_id=row_id, depth=depth) - for up, row_id, depth in wanted - held.keys() - ] - ) - - -def rebuild_search(service): - """Write each component's weighted search path. - - `A` is its own name, `B` its ancestors', `C` its service's. An - exact hit on a rollup then outranks a leaf matched only through its - service. + for incoming in components: + row, _ = ServiceComponent.objects.update_or_create( + service=service, + external_id=incoming.external_id, + **_signed( + author, + name=incoming.name, + status_page_order=incoming.order, + is_overall=incoming.is_overall, + is_archived=False, + archived_at=None, + ), + ) + rows[incoming.external_id] = row - Ancestor names are joined here, not in SQL. Postgres cannot build a - vector from a set of foreign keys. - """ - rows = list( - ServiceComponent.objects.filter(service=service).select_related("service") - ) - names = {row.id: row.name for row in rows} - chains = _chains(rows) - for row in rows: - ancestors = " ".join(names[up] for up in chains[row.id]) - row.search_document = ( - SearchVector(Value(row.name), weight="A") - + SearchVector(Value(ancestors), weight="B") - + SearchVector(Value(row.service.name), weight="C") + # Parents second: a child may arrive before its parent exists. + for incoming in components: + parent = ( + rows.get(incoming.parent_external_id) + if incoming.parent_external_id + else None ) - ServiceComponent.objects.bulk_update(rows, ["search_document"]) + row = rows[incoming.external_id] + if row.parent_id != (parent.id if parent else None): + row.parent = parent + row.updated_by = author + row.save(update_fields=["parent", "updated_by"]) + return rows def _archive_vanished(service, components, author): diff --git a/api/tests/factories.py b/api/tests/factories.py index 6ac6c51..27f75b7 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -89,15 +89,22 @@ def track(component, user=None): def ancestry(component): - """The ids above it, root first, read from the closure table. + """The ids above it, root first, worked out from `parent`. - `depth` counts the steps down to the component, so the largest is - the root. A breadcrumb reads the same order. + `ServiceComponent.ancestors` is the one walk up the tree, and a + breadcrumb reads the same order. """ - return list( - component.ancestor_links.order_by("-depth").values_list( - "ancestor_id", flat=True - ) + return [row.pk for row in component.ancestors] + + +def descendants(component): + """The ids under it, at any depth, the way `?ancestor=` asks.""" + from catalog.queries import descendant_ids + + return set( + ServiceComponent.objects.filter( + pk__in=descendant_ids(component.pk) + ).values_list("pk", flat=True) ) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 87d09bf..086e4bb 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -911,16 +911,14 @@ def component_form_data(staff_client, component, **fields): @pytest.mark.django_db -def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): +def test_reparenting_in_the_admin_moves_the_component(staff_client): """The admin posts a form, so it reaches the model by another road. - `ServiceComponent.save` rebuilds the derived columns. A component - reparented here must come out of it with ancestry and a search - vector. Without them it is invisible to `?ancestor=` and unfindable - by `?q=`. An untracked service is never polled to repair it. + A component reparented here has to answer `?ancestor=` under its + new parent at once. An untracked service is never polled, so + nothing else would come along and correct it. """ - from catalog.models import ServiceComponent - from tests.factories import ComponentFactory, ServiceFactory, ancestry + from tests.factories import ComponentFactory, ServiceFactory, ancestry, descendants service = ServiceFactory(name="Twilio") parent = ComponentFactory(service=service, name="Programmable Messaging") @@ -931,7 +929,7 @@ def test_reparenting_in_the_admin_rebuilds_the_derived_columns(staff_client): child.refresh_from_db() assert ancestry(child) == [parent.pk] - assert list(ServiceComponent.objects.search("programmable sms")) == [child] + assert descendants(parent) == {child.pk} @pytest.mark.django_db @@ -976,69 +974,24 @@ def bulk_delete(admin_client, components): @pytest.mark.django_db -def test_the_bulk_delete_rebuilds_the_ancestry(admin_client): - # `parent` is SET_NULL, so the leaf becomes a root. The cascade - # clears the deleted row's own links only. Without a rebuild the - # grandparent keeps a link to a grandchild it no longer sits above. - from polling.reconcile import rebuild_ancestry +def test_the_bulk_delete_leaves_no_stale_ancestry(admin_client): + # `parent` is SET_NULL, so the leaf becomes a root. A stored tree + # needed an admin override here. Without it the grandparent kept a + # link to a grandchild it no longer sits above. A computed one + # passes this trivially, which is why the override went. from tests.factories import ComponentFactory, ServiceFactory, ancestry service = ServiceFactory() top = ComponentFactory(service=service) middle = ComponentFactory(service=service, parent=top) leaf = ComponentFactory(service=service, parent=middle) - rebuild_ancestry(service) bulk_delete(admin_client, [middle]) + leaf.refresh_from_db() assert ancestry(leaf) == [] -@pytest.mark.django_db -def test_the_bulk_delete_rebuilds_the_search_path(admin_client): - # `search_document` holds each ancestor's name. Left stale, a - # deleted parent's name still finds a component under nothing. - from catalog.models import ServiceComponent - from polling.reconcile import rebuild_ancestry, rebuild_search - from tests.factories import ComponentFactory, ServiceFactory - - service = ServiceFactory(name="Twilio") - top = ComponentFactory(service=service, name="Programmable Messaging") - leaf = ComponentFactory(service=service, name="SMS", parent=top) - rebuild_ancestry(service) - rebuild_search(service) - - bulk_delete(admin_client, [top]) - - assert list(ServiceComponent.objects.search("messaging")) == [] - # The leaf still answers its own name. A rebuild that dropped the - # whole vector would pass the line above and serve nothing. - assert list(ServiceComponent.objects.search("sms")) == [leaf] - - -@pytest.mark.django_db -def test_the_bulk_delete_rebuilds_a_service_once(admin_client, monkeypatch): - # A rebuild rewrites every row of the service. Once per deleted row - # makes a selection of fifty cost fifty passes over the whole tree. - from catalog import admin as catalog_admin - from tests.factories import ComponentFactory, ServiceFactory - - service = ServiceFactory() - top = ComponentFactory(service=service) - first = ComponentFactory(service=service, parent=top) - second = ComponentFactory(service=service, parent=top) - - rebuilt = [] - monkeypatch.setattr( - catalog_admin, "rebuild_ancestry", lambda s: rebuilt.append(s.pk) - ) - monkeypatch.setattr(catalog_admin, "rebuild_search", lambda s: None) - - bulk_delete(admin_client, [first, second]) - - assert rebuilt == [service.pk] - - def related_item(admin_user, service, label): """One entry of a service row's View dropdown, count and link.""" from django.test import RequestFactory diff --git a/api/tests/test_archived.py b/api/tests/test_archived.py index 00f489c..3d58723 100644 --- a/api/tests/test_archived.py +++ b/api/tests/test_archived.py @@ -28,7 +28,6 @@ from authentication.models import User from dashboards.models import DashboardItem -from polling.reconcile import rebuild_ancestry from status.choices import EventKind, IncidentPhase from status.models import ServiceEvent from tests.conftest import jwt_client @@ -76,7 +75,6 @@ def descendants_of_a_component(): service = ServiceFactory() top = ComponentFactory(service=service) archived(service=service, parent=top) - rebuild_ancestry(service) url = reverse("component-list") return APIClient().get(url, {"ancestor": str(top.id)}).json()["results"] @@ -100,7 +98,6 @@ def a_components_path(): top = ComponentFactory(service=service) middle = archived(service=service, parent=top) leaf = ComponentFactory(service=service, parent=middle) - rebuild_ancestry(service) body = APIClient().get(reverse("component-detail", args=[leaf.id])).json() return [node for node in body["path"] if node["id"] == str(middle.id)] @@ -109,7 +106,6 @@ def a_components_descendant_count(): service = ServiceFactory() top = ComponentFactory(service=service) archived(service=service, parent=top) - rebuild_ancestry(service) body = APIClient().get(reverse("component-detail", args=[top.id])).json() return body["descendant_count"] diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index 587e34a..c4bb0ed 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -1,15 +1,14 @@ import pytest -from django.db.models import Q from django.db.utils import IntegrityError -from catalog.models import ComponentAncestor, Service, ServiceComponent, StatusPage -from polling.reconcile import rebuild_ancestry, rebuild_search +from catalog.models import Service, ServiceComponent, StatusPage from tests.factories import ( ComponentFactory, PollerFactory, ServiceFactory, UserFactory, ancestry, + descendants, track, watchers, ) @@ -194,46 +193,10 @@ def test_the_database_refuses_a_rollup_that_already_has_children(): parent.save() -def test_a_reparent_in_plain_python_rebuilds_the_derived_columns(db): - # An admin-only hook left every other writer stale: a shell session, - # a data migration, a later API write. The symptom is invisible. The - # row is missing from `?ancestor=`, from `?q=` and from every count. - service = ServiceFactory(name="Twilio") - parent = ComponentFactory(service=service, name="Programmable Messaging") - child = ComponentFactory(service=service, name="SMS") - - child.parent = parent - child.save() - - child.refresh_from_db() - assert ancestry(child) == [parent.pk] - assert list(ServiceComponent.objects.search("programmable sms")) == [child] - - -def test_a_move_between_services_rebuilds_the_service_left_behind(db): - # Only the new service was rebuilt. The old one kept rows naming - # this component in their ancestry and in their search documents. - old = ServiceFactory(name="Twilio") - new = ServiceFactory(name="Vonage") - top = ComponentFactory(service=old, name="Programmable Messaging") - leaf = ComponentFactory(service=old, name="SMS", parent=top) - rebuild_ancestry(old) - rebuild_search(old) - assert list(ServiceComponent.objects.search("programmable sms")) == [leaf] - - top.parent = None - top.service = new - top.save() - - leaf.refresh_from_db() - assert ancestry(leaf) == [] - assert list(ServiceComponent.objects.search("programmable sms")) == [] - - def test_a_parent_in_another_service_is_not_a_step_in_the_path(db): - # The closure table stops at the service boundary. A breadcrumb that - # walked past it names a component of another service. The same - # response then says the row has no ancestors. + # The walk stops at the service boundary. A breadcrumb that went + # past it names a component of another service. The same response + # then says the row has no ancestors. old = ServiceFactory(name="Twilio") top = ComponentFactory(service=old, name="Programmable Messaging") leaf = ComponentFactory(service=old, name="SMS", parent=top) @@ -247,85 +210,70 @@ def test_a_parent_in_another_service_is_not_a_step_in_the_path(db): assert leaf.path == "Twilio / SMS" -def test_a_save_that_moves_nothing_rebuilds_nothing(db, monkeypatch): - # A rebuild rewrites every row of the service. A save of an - # unrelated column must not pay for it, or one poll goes quadratic. - from polling import reconcile - - component = ComponentFactory() - calls = [] - monkeypatch.setattr( - reconcile, "rebuild_search", lambda service: calls.append(service) - ) - - component.is_featured = True - component.save() - - assert calls == [] - - -def test_ancestry_records_one_link_a_step_with_the_distance(db): - # A descendant query joins this table. Walking `parent` per row - # cannot use an index, and `for_display` only selects two levels up. +def test_the_tree_reads_every_step_not_one_level(db): + # A component's Components tab lists everything under it, and its + # breadcrumb names every step above it. One level either way would + # hide the grandchild and shorten the path. service = ServiceFactory() top = ComponentFactory(service=service) middle = ComponentFactory(service=service, parent=top) leaf = ComponentFactory(service=service, parent=middle) - rebuild_ancestry(service) # Root first, which is the order a breadcrumb reads. assert ancestry(leaf) == [top.id, middle.id] - # The distance, so one query can ask for the direct children alone. - # A parent is 1. Counting from the root instead would make `depth=1` - # name the top of the tree. - assert {link.ancestor_id: link.depth for link in leaf.ancestor_links.all()} == { - top.id: 2, - middle.id: 1, - } + assert descendants(top) == {middle.id, leaf.id} + # Nothing counts a component among the rows below it. + assert descendants(leaf) == set() - # Every descendant, not the direct children. `top` has two below it. - assert ServiceComponent.objects.filter(ancestor_links__ancestor=top).count() == 2 + +def test_the_tree_stops_at_the_service_boundary(db): + # Moving a component to another service leaves its old children + # pointing across. Counting them under the mover names rows that + # service's lists never hold. + old = ServiceFactory() + top = ComponentFactory(service=old) + leaf = ComponentFactory(service=old, parent=top) + + top.service = ServiceFactory() + top.save() + + assert descendants(top) == set() + leaf.refresh_from_db() + assert ancestry(leaf) == [] + + +def test_a_loop_in_the_parent_column_does_not_hang_the_tree(db): + # The column points at its own table, so bad data can make a cycle. + # Unguarded, the walk down never ends and the request never answers. + service = ServiceFactory() + first = ComponentFactory(service=service) + second = ComponentFactory(service=service, parent=first) + # `update` skips `clean`, which is the only thing refusing this. + ServiceComponent.objects.filter(pk=first.pk).update(parent=second) + + assert descendants(first) == {second.id} + first.refresh_from_db() + second.refresh_from_db() + assert ancestry(first) == [second.id] + assert ancestry(second) == [first.id] def test_deleting_a_component_takes_its_ancestry_with_it(db): - # An array of ids held no reference. Deleting a component left its - # id in every descendant, and only a poll of that service cleared - # it. An untracked service is never polled again. + # An array of ids held no reference, and a closure table needed a + # rebuild hook on four write paths. A computed answer passes this + # trivially, which is why the table went. service = ServiceFactory() top = ComponentFactory(service=service) middle = ComponentFactory(service=service, parent=top) leaf = ComponentFactory(service=service, parent=middle) - rebuild_ancestry(service) - # Read the key first. A delete clears it on the instance, so a - # filter written afterwards matches nothing and proves nothing. - gone = middle.pk middle.delete() - assert not ComponentAncestor.objects.filter( - Q(ancestor=gone) | Q(descendant=gone) - ).exists() - # `parent` is SET_NULL, so the leaf is now a root. The cascade - # clears the deleted row's own links only. Without the rebuild the - # leaf keeps a grandparent it no longer sits under. + # `parent` is SET_NULL, so the leaf is now a root. Nothing may + # still claim it sits under the grandparent. + leaf.refresh_from_db() assert ancestry(leaf) == [] - - -def test_deleting_a_component_takes_its_name_out_of_the_search_path(db): - # `search_document` holds each ancestor's name. Left stale, a - # deleted parent's name still finds a component under nothing. - service = ServiceFactory(name="Twilio") - top = ComponentFactory(service=service, name="Programmable Messaging") - leaf = ComponentFactory(service=service, name="SMS", parent=top) - rebuild_ancestry(service) - rebuild_search(service) - - top.delete() - - assert list(ServiceComponent.objects.search("messaging")) == [] - # The leaf still answers its own name. A rebuild that dropped the - # whole vector would pass the line above and serve nothing. - assert list(ServiceComponent.objects.search("sms")) == [leaf] + assert descendants(top) == set() def test_for_display_counts_descendants_without_a_query_a_row( @@ -337,7 +285,6 @@ def test_for_display_counts_descendants_without_a_query_a_row( top = ComponentFactory(service=service) middle = ComponentFactory(service=service, parent=top) ComponentFactory(service=service, parent=middle) - rebuild_ancestry(service) rows = list(ServiceComponent.objects.for_display()) with django_assert_num_queries(0): @@ -354,7 +301,6 @@ def test_the_descendant_count_leaves_out_an_archived_row(db): top = ComponentFactory(service=service) middle = ComponentFactory(service=service, parent=top) leaf = ComponentFactory(service=service, parent=middle) - rebuild_ancestry(service) leaf.is_archived = True leaf.save(update_fields=["is_archived"]) @@ -362,6 +308,21 @@ def test_the_descendant_count_leaves_out_an_archived_row(db): assert row.descendant_count == 1 +def test_the_descendant_count_walks_through_an_archived_row(db): + # An archived group is not counted, and it does not hide what sits + # under it either. A walk that stopped there would drop live rows + # the Components tab still lists. + service = ServiceFactory() + top = ComponentFactory(service=service) + middle = ComponentFactory(service=service, parent=top) + ComponentFactory(service=service, parent=middle) + middle.is_archived = True + middle.save(update_fields=["is_archived"]) + + row = ServiceComponent.objects.for_display().get(pk=top.pk) + assert row.descendant_count == 1 + + def test_a_component_read_without_for_display_has_no_count(db): # The count is an annotation and nothing falls back to a query. A # serving path that forgets `for_display` has to fail loudly, not @@ -418,55 +379,26 @@ def test_a_services_tracked_count_counts_a_component_once_per_owner(db): assert prepared.tracked_component_count == 1 -def test_the_audit_log_holds_no_search_vector(): - # `rebuild_search` writes the column in a bulk update, which records - # no history. Every old row would carry the vector of the last - # ordinary save. That describes neither the row nor the edit. - columns = {field.name for field in ServiceComponent.history.model._meta.fields} - assert "search_document" not in columns - - -def test_search_matches_a_components_path_and_ranks_the_rollup_first(db): - # The rollup's own name is an exact, weight-A hit. A leaf matches - # only through its service, at weight C, so it must sort below. +def test_search_reaches_a_component_through_its_services_name(db): + # "twilio" has to find SMS. Its own name says nothing about which + # service publishes it. A name-only match returns an empty page for + # the word people actually type. service = ServiceFactory(name="Twilio") rollup = ComponentFactory(service=service, name="Twilio", is_overall=True) parent = ComponentFactory(service=service, name="Programmable Messaging") leaf = ComponentFactory(service=service, name="SMS", parent=parent) - rebuild_ancestry(service) - rebuild_search(service) - found = list(ServiceComponent.objects.search("twilio")) - assert found[0] == rollup - assert leaf in found + assert set(ServiceComponent.objects.search("twilio")) == {rollup, parent, leaf} # OR semantics would let the rollup match too: it says "twilio" but # never "sms". assert list(ServiceComponent.objects.search("twilio sms")) == [leaf] -def test_search_weights_a_components_own_name_over_its_ancestors_over_its_service(db): - # A single-service fixture cannot separate the weights. Every - # component in it shares one service name, so A and C score the - # same. Three services isolate each weight in turn. - own_service = ServiceFactory(name="Widgets") - own_name = ComponentFactory(service=own_service, name="Twilio") - - ancestor_service = ServiceFactory(name="Gadgets") - ancestor = ComponentFactory(service=ancestor_service, name="Twilio") - via_ancestor = ComponentFactory( - service=ancestor_service, name="Portal", parent=ancestor - ) - rebuild_ancestry(ancestor_service) - rebuild_search(ancestor_service) - - via_service = ServiceFactory(name="Twilio") - via_service_component = ComponentFactory(service=via_service, name="Messaging") - - rebuild_search(own_service) - rebuild_search(via_service) +def test_search_ignores_the_case_and_matches_part_of_a_word(db): + # People type a fragment in lower case. An exact match would answer + # nothing until the whole name was typed the provider's way. + service = ServiceFactory(name="Twilio") + leaf = ComponentFactory(service=service, name="Programmable Messaging") - ranked = {row.id: row.rank for row in ServiceComponent.objects.search("twilio")} - assert ( - ranked[own_name.id] > ranked[via_ancestor.id] > ranked[via_service_component.id] - ) + assert list(ServiceComponent.objects.search("messag")) == [leaf] diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py index 654c941..b4d175d 100644 --- a/api/tests/test_component_api.py +++ b/api/tests/test_component_api.py @@ -3,7 +3,6 @@ from django.utils import timezone from rest_framework.test import APIClient -from polling.reconcile import rebuild_ancestry, rebuild_search from status.choices import EventKind, IncidentPhase, Severity, StatusSource from status.models import ComponentStatus, ServiceEvent from tests.factories import ComponentFactory, ServiceFactory, track @@ -33,8 +32,6 @@ def tree(): leaf = ComponentFactory( service=service, name="SMS", parent=parent, status_page_order=1 ) - rebuild_ancestry(service) - rebuild_search(service) return service, rollup, parent, leaf @@ -126,10 +123,10 @@ def test_the_detail_refuses_an_archived_component(client, tree): assert client.get(reverse("component-detail", args=[leaf.id])).status_code == 404 -def test_a_query_keeps_the_rank_the_search_worked_out(client, tree): - # The rollup carries the word in its own name. The other two match - # only through their service. A default sort applied after the - # search discards the rank, and then the list reads alphabetically. +def test_a_query_leads_with_the_service_rollup(client, tree): + # Typing a service's name matches every part of it. Without the + # rollup first, eighty rows bury the one row that answers the + # question the reader asked. response = client.get(reverse("component-list"), {"q": "twilio"}) assert response.json()["results"][0]["name"] == "Twilio" @@ -193,35 +190,6 @@ def test_event_narrows_to_the_components_one_event_affects(client, tree): assert [r["name"] for r in response.json()["results"]] == ["Programmable Messaging"] -def test_a_ranked_search_pages_without_repeating_a_row(client): - """`?q=` is Discover's default path, and `rank` ties heavily. - - Identical documents rank identically. A page boundary inside a run - of ties is where a cursor repeats a row or skips one. Infinite - scroll then shows the fault. - """ - service = ServiceFactory(name="Twilio") - for _ in range(7): - ComponentFactory(service=service, name="SMS") - rebuild_ancestry(service) - rebuild_search(service) - - seen, pages = [], 0 - url = reverse("component-list") - params = {"q": "twilio sms", "page_size": 2} - while url: - pages += 1 - # Seven rows, two a page. A cursor that repeats a page would - # otherwise hang the suite rather than fail it. - assert pages <= 4, "paging did not reach the end" - body = client.get(url, params).json() - seen += [row["id"] for row in body["results"]] - url, params = body["next"], {} - - assert seen == list(dict.fromkeys(seen)), "a row came back on two pages" - assert len(seen) == 7, "a row was never reached" - - def test_the_path_leaves_out_an_archived_ancestor(client): # Each node carries an id a client links to, and the detail answers # 404 for an archived component. A breadcrumb step pointing at one diff --git a/api/tests/test_migrations.py b/api/tests/test_migrations.py deleted file mode 100644 index 0acddb4..0000000 --- a/api/tests/test_migrations.py +++ /dev/null @@ -1,96 +0,0 @@ -"""The data migration that fills the ancestry table. - -It runs once. A service nobody polls is never rebuilt, so what this -writes is what that service keeps. - -The function is called directly, against the live app registry. It -reads `parent`, which the models still hold, so a historical registry -would answer the same. -""" - -from importlib import import_module - -import pytest - -from catalog.models import ComponentAncestor -from tests.factories import ComponentFactory, ServiceFactory, ancestry - -fill_from_parent = import_module( - "catalog.migrations.0008_component_ancestor_table" -).fill_from_parent - -pytestmark = pytest.mark.django_db - - -def backfill(): - """Run the migration against a table it has not seen.""" - from django.apps import apps - - ComponentAncestor.objects.all().delete() - fill_from_parent(apps, None) - - -def test_the_backfill_writes_the_chain_of_every_component(): - # Nothing else fills the table on a service nobody polls. An empty - # backfill leaves every such component with no descendants and no - # breadcrumb, forever. - service = ServiceFactory() - top = ComponentFactory(service=service) - middle = ComponentFactory(service=service, parent=top) - leaf = ComponentFactory(service=service, parent=middle) - - backfill() - - assert ancestry(top) == [] - assert ancestry(middle) == [top.id] - assert ancestry(leaf) == [top.id, middle.id] - - -def test_the_backfill_gives_a_row_whose_parent_went_no_ancestors(): - # `parent` is SET_NULL, so deleting a component makes its children - # roots. Keeping the steps above the gap answers `?ancestor=` with a - # tree nobody sits in. It also inflates the top row's count. - service = ServiceFactory() - top = ComponentFactory(service=service) - middle = ComponentFactory(service=service, parent=top) - leaf = ComponentFactory(service=service, parent=middle) - middle.delete() - - backfill() - - leaf.refresh_from_db() - assert ancestry(leaf) == [] - assert top.descendant_links.count() == 0 - - -def test_the_backfill_stops_at_the_service_boundary(): - # Moving a component to another service leaves its old children - # pointing across. A step there names a row the old service's lists - # never hold. - old = ServiceFactory() - top = ComponentFactory(service=old) - leaf = ComponentFactory(service=old, parent=top) - top.service = ServiceFactory() - top.save() - - backfill() - - assert ancestry(leaf) == [] - - -def test_a_loop_in_the_parent_column_does_not_hang_the_backfill(): - # The column points at its own table, so bad data can make a cycle. - # Without the guard the migration never ends, and the deploy that - # runs it never finishes. - from catalog.models import ServiceComponent - - service = ServiceFactory() - first = ComponentFactory(service=service) - second = ComponentFactory(service=service, parent=first) - # `update` skips `clean`, which is the only thing refusing this. - ServiceComponent.objects.filter(pk=first.pk).update(parent=second) - - backfill() - - assert ancestry(first) == [second.id] - assert ancestry(second) == [first.id] diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index e4aeaee..1cfd146 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -7,7 +7,7 @@ NormalisedEvent, NormalisedUpdate, ) -from polling.reconcile import apply_fetch, rebuild_ancestry, rebuild_search +from polling.reconcile import apply_fetch from status.choices import ( EventKind, EventSource, @@ -297,52 +297,10 @@ def test_a_second_poll_does_not_rewrite_who_made_the_row(): @pytest.mark.django_db -def test_a_poll_writes_the_ancestor_chain(): - # Nothing else writes the column. Without this pass a component's - # Components tab lists no descendant, whatever the provider nests. - service = ServiceFactory() - apply_fetch( - service, - [_component(external_id="a"), _component(external_id="b", parent="a")], - [], - StatusSource.PROVIDER, - ) - - top = ServiceComponent.objects.get(external_id="a") - child = ServiceComponent.objects.get(external_id="b") - assert ancestry(top) == [] - assert ancestry(child) == [top.id] - - -@pytest.mark.django_db -def test_a_poll_rebuilds_the_service_once_not_once_a_component(monkeypatch): - # `ServiceComponent.save` rebuilds the whole service, and this pass - # saves every component of it. Unsuppressed, a hundred components - # cost a hundred rebuilds of a hundred rows, on every poll. - from polling import reconcile - - calls = [] - real = reconcile.rebuild_ancestry - monkeypatch.setattr( - reconcile, - "rebuild_ancestry", - lambda service: calls.append(service) or real(service), - ) - - service = ServiceFactory() - rows = [_component(external_id=str(n)) for n in range(5)] - apply_fetch(service, rows, [], StatusSource.PROVIDER) - assert len(calls) == 1 - - calls.clear() - apply_fetch(service, rows, [], StatusSource.PROVIDER) - assert len(calls) == 1 - - -@pytest.mark.django_db -def test_a_reparent_rewrites_the_subtree(): - # The chain of a grandchild changes when its parent moves, and - # nothing tells the grandchild. The pass rewrites the service. +def test_a_reparent_moves_the_whole_subtree(): + # A provider can lift a group out from under another. The poll + # writes one parent, and the grandchild below it has to read its + # new place without a second write. service = ServiceFactory() nested = [ _component(external_id="a"), @@ -361,44 +319,3 @@ def test_a_reparent_rewrites_the_subtree(): assert ancestry(ServiceComponent.objects.get(external_id="c")) == [ ServiceComponent.objects.get(external_id="b").id ] - - -@pytest.mark.django_db -def test_a_loop_in_the_parent_column_does_not_hang_the_pass(): - # The column points at its own table, so bad data can make a cycle. - # Without the guard the walk never reaches a root, and the worker - # never finishes the poll that called it. - service = ServiceFactory() - first = ComponentFactory(service=service) - second = ComponentFactory(service=service, parent=first) - # `update` skips `clean`, which is the only thing refusing this. - ServiceComponent.objects.filter(pk=first.pk).update(parent=second) - - rebuild_ancestry(service) - - assert ancestry(first) == [second.id] - assert ancestry(second) == [first.id] - - -@pytest.mark.django_db -def test_a_reparent_rewrites_every_descendants_path(): - # Ancestry and the search vector both carry the chain. A provider - # moving one node must not leave its children searchable under - # where they used to sit. - service = ServiceFactory(name="Acme") - old = ComponentFactory(service=service, name="Legacy", external_id="old") - new = ComponentFactory(service=service, name="Platform", external_id="new") - leaf = ComponentFactory( - service=service, name="Queue", external_id="leaf", parent=old - ) - rebuild_ancestry(service) - rebuild_search(service) - - leaf.parent = new - leaf.save(update_fields=["parent"]) - rebuild_ancestry(service) - rebuild_search(service) - - assert ancestry(leaf) == [new.id] - assert list(ServiceComponent.objects.search("legacy queue")) == [] - assert list(ServiceComponent.objects.search("platform queue")) == [leaf] diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 45529ce..271f0dc 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -88,7 +88,7 @@ info: Query parameters are declared in a `FilterSet`. Almost none is generated from a column of the same name. `is_overall` and `kind` are the two that are. `q` is no filter at all. - `ComponentListView.filter_queryset` ranks on it before the backends run. + `ComponentListView.filter_queryset` narrows on it before the backends run. So a parameter is not an ORM path, and a field's path in the response body does not build its filter. `service` takes a slug where the body carries a nested `ServiceRef`. `component` @@ -306,9 +306,9 @@ paths: `service`, a component's descendants with `ancestor`, an event's affected list with `event`. - `q` matches a component's whole path, so `twilio` finds `SMS`. Results are then ranked by - relevance. Without `q` the default is `suggested`, except under `service`, where it is - `status_page_order`. + `q` matches part of a component's name or its service's name, so `twilio` finds `SMS`. + Several words are an AND. The service rollup leads the results. Without `q` the default + is `suggested`, except under `service`, where it is `status_page_order`. Archived components are never listed. An event whose only affected component was archived returns an empty page. diff --git a/docs/check_docs.py b/docs/check_docs.py index f6529ef..5dd4342 100755 --- a/docs/check_docs.py +++ b/docs/check_docs.py @@ -62,7 +62,7 @@ "Poller.cooldown_seconds": "own override or /meta/ default", "Service.poller": "OneToOneField", "Component.path": "walk of parent", - "Component.descendant_count": "Count of ComponentAncestor where ancestor is this id", + "Component.descendant_count": "recursive walk down parent, archived rows left out", "Component.is_tracked": "per-user annotation over DashboardItem", "Component.upcoming_maintenance": "soonest unfinished maintenance event across that M2M", "Component.upcoming_maintenance_count": "Count of unfinished maintenance across that M2M", From 22939c97e6e51e253e21718761f4ddce54e7393b Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 14:59:57 -0700 Subject: [PATCH 57/94] docs: the specs argue for computed ancestry and substring search The client design argued for a closure table, having been rewritten once already from arguing for an array. Both stored the answer to a question `parent` answers, and each needed four writers to stay true. The section now says so, with the measurements, and names the sync obligation as the reason rather than speed. Section 8 argued for a weighted vector. Weight `B` made a component's document depend on other rows, and that is what forced the subtree rewrite. The section now describes a per-word substring match, and says plainly what it gives up. The v1 spec's ER diagram loses the table and the column. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- docs/specs/2026-08-23-statusboard-design.md | 25 ++-- .../2026-09-03-statusboard-client-design.md | 109 ++++++++++-------- 2 files changed, 69 insertions(+), 65 deletions(-) diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 4ba12c4..125b947 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -76,8 +76,7 @@ what makes sharing a change to `dashboards` alone. ## 3. Data model Every model extends `common.BaseModel`: UUID primary key, `created_at`, `updated_at`, -`created_by`, `updated_by`, `ordering = ["-created_at"]`. `ComponentAncestor` is the one -exception: reconcile derives every row from `parent`, so an author records nothing. +`created_by`, `updated_by`, `ordering = ["-created_at"]`. `auth.Permission` and `ContentType` keep integer keys — that is Django, not a choice. ### authentication @@ -155,18 +154,11 @@ erDiagram string external_id "unique with service" string name uuid parent_id FK "self, null at top level" - tsvector search_document "weighted path, written by reconcile" int status_page_order bool is_overall bool is_featured "first key of the suggested sort" datetime archived_at "set when it stops being published" } - ComponentAncestor { - uuid id PK - uuid ancestor_id FK "the component above" - uuid descendant_id FK "the component below, unique with ancestor" - int depth "steps down to the descendant, a parent is 1" - } ComponentStatus { uuid id PK uuid component_id FK @@ -218,8 +210,6 @@ erDiagram Service ||--|| Poller : "read by" Service ||--o{ ServiceComponent : publishes ServiceComponent ||--o| ServiceComponent : "parent of" - ServiceComponent ||--o{ ComponentAncestor : "sits above" - ServiceComponent ||--o{ ComponentAncestor : "sits below" ServiceComponent ||--o{ ComponentStatus : "severity over time" Poller ||--o{ PollRun : "attempts" Service ||--o{ ServiceEvent : "published on" @@ -227,9 +217,9 @@ erDiagram ServiceEvent }o--o{ ServiceComponent : affects ``` -Every model above but `ComponentAncestor` also carries `common.BaseModel`: `created_at`, -`updated_at`, `created_by`, `updated_by`. They are omitted so the diagram shows what -distinguishes each table. +Every model above also carries `common.BaseModel`: `created_at`, `updated_at`, +`created_by`, `updated_by`. They are omitted so the diagram shows what distinguishes +each table. Reading it: @@ -237,10 +227,9 @@ Reading it: these, so the arrow from `DashboardItem` has one destination. - **`StatusPage` is separate from `Service`** because it is the *source*, and it is where the unique URL and the poller's state live. -- **`ComponentAncestor` is the tree, flattened**, one row per pair with one component above the - other. `parent` alone answers one level; this answers any depth in a single indexed join, which - is what `?ancestor=` and every descendant count ask for. Both columns are real foreign keys, so - deleting a component takes every claim about where it sat with it. +- **`parent` is the whole tree.** `?ancestor=` and every descendant count read any depth from + it, with one recursive query. A stored copy of the answer needed a writer on every path that + could move a component, and it was wrong until that writer ran. - **`ComponentStatus` is append-only**, one row per severity change, with a partial unique index marking the current one. Current state and history are the same table. diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md index fd58de3..b60cb11 100644 --- a/docs/specs/2026-09-03-statusboard-client-design.md +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -242,9 +242,9 @@ Every one of them is labelled Smart in the interface. popularity, the same as the service sort it replaces. A middling component that is broken now beats a popular one that is fine. -There is one sort control and one label, **Smart**, on every list. With a `q` it ranks by -`SearchRank`. Without one it is `suggested`. A separate "Best match" would name a ranking -that does not exist until you type. +There is one sort control and one label, **Smart**, on every list. With a `q` it is +`(-is_overall, *suggested)`, so the service rollup leads. Without one it is `suggested`. A +separate "Best match" would name a ranking the search does not work out. ## 6. Model changes @@ -274,37 +274,53 @@ once a URL was triaged. for copying `PollRun.error`, which can disagree with its source. Here there is no source. The counter is the record. -`catalog.ComponentAncestor`, the component tree flattened. +### Ancestry -| Field | Note | -| --- | --- | -| `ancestor` | FK to `ServiceComponent`, the one above | -| `descendant` | FK to `ServiceComponent`, the one below. Unique with `ancestor`. | -| `depth` | the steps down from `ancestor` to `descendant`. A parent is 1. | +**No model holds it.** `parent` already says where a component sits, and +`?ancestor=`, `descendant_count` and the breadcrumb are all read from it. + +A flattened `ComponentAncestor` table was built for this and then removed. So was an array +of ancestor ids before it. Each stored the answer to a question `parent` already answers, +and a stored answer has to be rewritten every time the tree moves. That obligation reached +`ServiceComponent.save`, `ServiceComponent.delete`, the admin's `delete_queryset`, and a +`held_rebuilds()` context manager suppressing all three during a poll. Miss one write path +and the component is silently absent from `?ancestor=` and from every count. An untracked +service is never polled, so nothing comes along and repairs it. + +**Speed was not the reason to store it, and it is not the reason to stop.** The claim was +that walking `parent` at query time cannot use an index. That is true of a Python loop +asking one query a level. It is false of a recursive CTE, which is one indexed query. +Measured on 341 components, deeper than a real service: + +| Read | Closure table | Recursive CTE | +| --- | --- | --- | +| every descendant of a node | 0.56 ms | 0.73 ms | +| one breadcrumb, root first | 0.21 ms | 0.35 ms | +| `descendant_count`, page of 50 | 0.42 ms | 0.50 ms | -One row per pair, written by reconcile. `parent` answers one level. This answers any depth -in one indexed join, which is what `?ancestor=` and every `descendant_count` ask for. A -breadcrumb reads the same rows the other way round, root first by `-depth`. +A fifth of a millisecond, against four writers that could each leave the answer wrong. The +sync obligation is what the change buys back. -**Both columns are foreign keys, and that is the point.** An array of ancestor ids buys the -same indexed test and holds no reference. `parent` is `SET_NULL`, so nothing cleans an array -when a component goes, and every descendant keeps naming a row that is gone. Only a poll of -that service repairs it, and an untracked service is never polled again. A foreign key -cascades in the same statement as the delete. +`catalog.queries` holds the walk down, as one SQL definition that `?ancestor=` and +`descendant_count` share. Three rules are in it: -It is not a `BaseModel`. Reconcile derives every row from `parent`, so an author and an edit -time record nothing anybody reads. The UUIDv7 key stays. +- **A cycle guard.** `parent` is a self-FK and nothing forbids a loop. The walk carries the + ids it has visited and refuses to step onto one twice. A depth ceiling would also stop a + loop, and it would cut a deep tree that is not one. +- **The service boundary.** Each step tests `service_id`. Moving a component leaves its old + children pointing across, and a step there names a row that service's lists never hold. +- **Archived rows.** A count leaves them out, because no list serves one. The walk still + passes through them, or an archived group would hide the live rows under it. -No row says a component is its own ancestor. Every reader counts other components, so a -depth-zero row would add one to every answer. Two check constraints hold that: one forbids -the row, one forbids the distance that would describe it. +`descendant_count` is a correlated subquery annotated by `for_display`, so a page of fifty +costs one query for it and not fifty. The breadcrumb stays a Python walk up `parent`: +`for_display` already selects two levels of it, so the common case costs nothing. ### New columns | Column | Type | Written by | | --- | --- | --- | | `ServiceComponent.is_featured` | bool | admin | -| `ServiceComponent.search_document` | `SearchVectorField`, GIN index | reconcile | | `ServiceEvent.detected_by` | `provider` or `system` | set when the event opens, never changed | | `EventUpdate.source` | `provider` or `system` | the adapter, or reconcile | @@ -428,30 +444,29 @@ the same question. ## 8. Search -`q` matches a component's full path. Searching `twilio` finds `SMS`, because Twilio is in -its path. Searching `twilio sms` finds it too. - -`ServiceComponent.search_document` is a `SearchVectorField` with a GIN index, written in -the reconcile pass that already sets names and parents. A path built by walking `parent` at -query time cannot use an index. - -| Weight | Source | -| --- | --- | -| `A` | the component's own name | -| `B` | its ancestors' names, walked from `parent` in that pass | -| `C` | its service's name | - -So `twilio` ranks the Twilio rollup first, then top-level components, then leaves. The -ranking solves most of the flooding a popular service would cause. `q` becomes a -`SearchQuery` in websearch mode, so several words are an AND, ordered by `SearchRank`. - -**Renaming or reparenting rewrites a whole subtree.** `ComponentAncestor` and -`search_document` both carry ancestry, so both change. Providers do move components, so that -rewrite belongs inside the reconcile pass. This is the one place the design fans out. - -Both read one walk of `parent`, over the rows the pass already holds. A deep tree costs no -extra query. The ancestry rebuild then writes only the links that moved, because almost no -poll moves a component. +`q` matches part of a component's own name or its service's name. Searching `twilio` finds +`SMS`, because Twilio publishes it. Searching `twilio sms` finds it too: every word must +match, and either column may satisfy it. Matching ignores case. + +It is `icontains` per word, not a `tsvector`. Measured on 4,800 components across 60 +services, larger than realistic, a stored weighted vector with a GIN index answers in about +1.5 ms less. That is the whole return on the apparatus it needs. + +The apparatus was the cost. A component's vector held its ancestors' names at weight `B`, +so the vector depended on **other rows**. Renaming one group rewrote every document below +it, and that subtree rewrite is what forced a rebuild onto `save`, onto `delete`, onto the +admin's bulk delete, and into a `held_rebuilds()` suppressor for polls. With no weights +there is nothing to keep in step. `ServiceComponent` becomes a plain model with a `parent`. + +**What is given up**, plainly: + +- **Stemming.** `outages` no longer finds `Outage`. A substring match covers a prefix, not + a different ending. +- **Relevance ranking.** There is no score, so nothing sorts by closeness of match. The + rollup leads instead, by `-is_overall`, which is what stops a popular service flooding + the list with its eighty parts. +- **Matching through an intermediate group.** A leaf is no longer found by the name of a + group above it, only by its own name and its service's. The service `search_fields` go with the service list. From 6c3fe1bfeab8546c97be9e1b592553a821d5bd54 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:13:17 -0700 Subject: [PATCH 58/94] refactor: drop the is_featured fieldset description The text explained featuring in the middle of the change form, where it read as instructions rather than a code comment. The field name and its heading already say what it does. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index d0d31e4..d9aea02 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -556,17 +556,7 @@ class ServiceComponentAdmin( _("Position"), {"fields": ["parent", "status_page_order", "is_overall"]}, ), - ( - _("Featuring"), - { - "fields": ["is_featured"], - "description": _( - "Leads the suggested sort on the catalog. Tick it on any " - "component. A rollup surfaces its service, and a leaf " - "surfaces that part." - ), - }, - ), + (_("Featuring"), {"fields": ["is_featured"]}), audit_section(), ] inlines = [ComponentStatusInline] From 97222e54de577eee5aa36ea52da287503fdad0e8 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:14:35 -0700 Subject: [PATCH 59/94] refactor: replace the two feature actions with list_editable A BooleanField needs no custom action. list_editable renders it as a checkbox in the changelist, with one Save. That also fixes what the actions could never do: feature some rows and unfeature others in one pass, since a mixed selection defeats a single toggle action. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 36 ++----------------- api/tests/test_admin.py | 76 +++++++++++++---------------------------- 2 files changed, 27 insertions(+), 85 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index d9aea02..f8e6f8f 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -539,6 +539,9 @@ class ServiceComponentAdmin( autocomplete_scope = ("service",) # The date follows the flag, so it is shown and never typed. readonly_fields = ["archived_at"] + # A checkbox in the list, not an action. One Save flips any mix of + # rollups and leaves, which a toggle action could never do at once. + list_editable = ["is_featured"] fieldsets = [ ( None, @@ -560,11 +563,6 @@ class ServiceComponentAdmin( audit_section(), ] inlines = [ComponentStatusInline] - # Two, not a toggle. A selection can mix already-featured and not, - # and a toggle would leave an admin unable to predict the result. - # `PollerAdmin.toggle_pause` can be one action because it only ever - # acts on a single object, which has one state to flip. - actions = ["feature_selected", "unfeature_selected"] def get_form(self, request, obj=None, **kwargs): # The parent picker needs to know which service is asking. @@ -657,34 +655,6 @@ def display_severity(self, obj): def watchers(self, obj): return obj.watcher_count - @admin.action(description=_("Feature selected components")) - def feature_selected(self, request, queryset): - """Feature every selected component, rollup or leaf. - - The change form above sets the same flag one row at a time. - Both are unrestricted, because `suggested` reads the flag on - every component and a featured leaf is a real editorial choice. - """ - count = 0 - for component in queryset.filter(is_featured=False): - component.is_featured = True - component.save(update_fields=["is_featured"]) - count += 1 - self.message_user( - request, _("Featured %(count)d component(s).") % {"count": count} - ) - - @admin.action(description=_("Unfeature selected components")) - def unfeature_selected(self, request, queryset): - count = 0 - for component in queryset.filter(is_featured=True): - component.is_featured = False - component.save(update_fields=["is_featured"]) - count += 1 - self.message_user( - request, _("Unfeatured %(count)d component(s).") % {"count": count} - ) - @admin.register(ServiceRequest) class ServiceRequestAdmin(ModelAdmin): diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 086e4bb..37e9e14 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -662,72 +662,44 @@ def test_service_requests_are_listed_by_demand(admin_client, db): @pytest.mark.django_db -def test_the_feature_action_flips_the_flag(admin_client, db): - # Selecting a component and running the action is how the spec's - # "bulk action" is meant to be used. A missing action leaves an - # admin with no way to feature many components at once. +def test_the_component_list_offers_the_featured_checkbox(admin_client, db): + # list_editable renders the flag as a checkbox in the row. Without + # it, featuring more than one component takes a form visit each. from tests.factories import ComponentFactory component = ComponentFactory(is_featured=False, is_overall=True) - admin_client.post( - reverse("admin:catalog_servicecomponent_changelist"), - {"action": "feature_selected", "_selected_action": [str(component.pk)]}, - ) - component.refresh_from_db() - assert component.is_featured is True - - -@pytest.mark.django_db -def test_the_unfeature_action_flips_it_back(admin_client, db): - # The reverse has to exist too. Otherwise featuring by mistake is - # permanent, short of editing the row on its change page. - from tests.factories import ComponentFactory - - component = ComponentFactory(is_featured=True, is_overall=True) - admin_client.post( - reverse("admin:catalog_servicecomponent_changelist"), - {"action": "unfeature_selected", "_selected_action": [str(component.pk)]}, - ) - component.refresh_from_db() - assert component.is_featured is False - - -@pytest.mark.django_db -def test_the_feature_action_features_a_leaf(admin_client, db): - # Discover lists every component, and `suggested` reads the flag on - # every row. Featuring one part of a service is an editorial choice - # an admin is allowed to make. - from tests.factories import ComponentFactory - - leaf = ComponentFactory(is_featured=False, is_overall=False) - admin_client.post( - reverse("admin:catalog_servicecomponent_changelist"), - {"action": "feature_selected", "_selected_action": [str(leaf.pk)]}, - ) - leaf.refresh_from_db() - assert leaf.is_featured is True + response = admin_client.get(reverse("admin:catalog_servicecomponent_changelist")) + assert response.status_code == 200 + assert b'name="form-0-is_featured"' in response.content + assert str(component.pk).encode() in response.content @pytest.mark.django_db -def test_the_feature_action_takes_a_mixed_selection_whole(admin_client, db): - # A rollup and a leaf are both featurable, so a selection holding - # one of each leaves nothing behind. +def test_the_component_list_formset_flips_the_flag(admin_client, db): + # A mixed selection defeats the two actions in one pass. One row is + # featured, the other cleared, one Save. from tests.factories import ComponentFactory, ServiceFactory service = ServiceFactory() - overall = ComponentFactory(service=service, is_overall=True, is_featured=False) - leaf = ComponentFactory(service=service, is_overall=False, is_featured=False) + to_feature = ComponentFactory(service=service, is_overall=True, is_featured=False) + to_clear = ComponentFactory(service=service, is_overall=False, is_featured=True) admin_client.post( reverse("admin:catalog_servicecomponent_changelist"), { - "action": "feature_selected", - "_selected_action": [str(overall.pk), str(leaf.pk)], + "form-TOTAL_FORMS": "2", + "form-INITIAL_FORMS": "2", + "form-MIN_NUM_FORMS": "0", + "form-MAX_NUM_FORMS": "1000", + "form-0-id": str(to_feature.pk), + "form-0-is_featured": "on", + "form-1-id": str(to_clear.pk), + "_save": "Save", }, ) - overall.refresh_from_db() - leaf.refresh_from_db() - assert overall.is_featured is True - assert leaf.is_featured is True + to_feature.refresh_from_db() + to_clear.refresh_from_db() + assert to_feature.is_featured is True + assert to_clear.is_featured is False @pytest.mark.django_db From d91aa506f8c8167e8b7d8bc74913d5e297229ef5 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:15:35 -0700 Subject: [PATCH 60/94] fix: link service requests from the admin sidebar ServiceRequestAdmin answered, but nothing in the sidebar pointed to it. Every existing test only checked the changelist responds, so the gap shipped unseen until a human went looking for the page by eye. Adds the sixth link, and a test that reads the sidebar titles directly so a page missing from navigation fails the suite next time. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/settings.py | 5 +++++ api/tests/test_admin.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/api/api/settings.py b/api/api/settings.py index 4b21865..3152fe6 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -367,6 +367,11 @@ "icon": "person", "link": reverse_lazy("admin:authentication_user_changelist"), }, + { + "title": _("Service requests"), + "icon": "playlist_add", + "link": reverse_lazy("admin:catalog_servicerequest_changelist"), + }, ], }, ], diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 37e9e14..b9499dd 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -661,6 +661,28 @@ def test_service_requests_are_listed_by_demand(admin_client, db): assert response.content.index(b"b.example") < response.content.index(b"a.example") +def test_the_sidebar_names_every_top_level_destination(): + # A changelist that answers is not one anyone can find. The sidebar + # is the only listing browsed without a URL. Every top-level page + # needs a title in it. + from django.conf import settings + + titles = { + str(item["title"]) + for group in settings.UNFOLD["SIDEBAR"]["navigation"] + for item in group["items"] + } + for title in ( + "Dashboard", + "Services", + "Polling", + "Boards", + "Users", + "Service requests", + ): + assert title in titles, f"{title} has no sidebar entry" + + @pytest.mark.django_db def test_the_component_list_offers_the_featured_checkbox(admin_client, db): # list_editable renders the flag as a checkbox in the row. Without From 072501b0eba4cee05951b66c1fbdfe16eed568b7 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:25:55 -0700 Subject: [PATCH 61/94] fix: an import records the person who asked for it The importer signed every service with the system account. Both request paths carry a caller: the admin's Import-from-URL action and POST /catalog/import/. Each lost it, so an admin who imported Twilio read as the bot. `import_from_url` now takes an author and falls back to the system account. `seed_dev` has no request, so it keeps the fallback. An anonymous import records null, the answer a service request already gives. The bot then means automated, and nothing else. A Poller inherits the author of the service that spawned it, for the same reason. It falls back to the system account only when the service itself has none. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 4 +++- api/catalog/models.py | 7 ++++--- api/catalog/views/imports.py | 6 +++++- api/polling/importer.py | 25 ++++++++++++++++--------- api/tests/test_admin.py | 26 ++++++++++++++++++++++++-- api/tests/test_import.py | 29 +++++++++++++++++++++++++++++ api/tests/test_polling.py | 21 +++++++++++++++++++-- api/tests/test_service_requests.py | 2 +- 8 files changed, 101 insertions(+), 19 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index f8e6f8f..210e808 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -457,7 +457,9 @@ def import_from_status_page(self, request, form): url = form.cleaned_data["status_page_url"] try: - service, created = import_from_url(url) + # The staff member who pasted the URL, the same stamp an + # add form leaves. The system account means automated. + service, created = import_from_url(url, author=request.user) except Exception as error: # noqa: BLE001 — the page is a stranger's # Anything can come back from a URL a person pasted. Say what # happened rather than showing a 500. diff --git a/api/catalog/models.py b/api/catalog/models.py index 89ae060..94bc64d 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -106,9 +106,10 @@ def ensure_poller(self): # save. This is the smaller of the two. from polling.models import Poller - # Nobody adds this one, so the system account signs it. A blank - # author would read as one that was lost. - author = get_user_model().objects.system() + # A person caused the service, so the poller they spawned + # carries them. The system account signs it only when the + # service has none. A blank author reads as one that was lost. + author = self.created_by or get_user_model().objects.system() poller, _ = Poller.objects.get_or_create( service=self, defaults={"created_by": author, "updated_by": author}, diff --git a/api/catalog/views/imports.py b/api/catalog/views/imports.py index ad082e7..7702d01 100644 --- a/api/catalog/views/imports.py +++ b/api/catalog/views/imports.py @@ -53,8 +53,12 @@ def post(self, request): if not body.is_valid(): return Response(body.errors, status=400) url = body.validated_data["status_page_url"] + # Anonymous stays anonymous, as it does on a service request. + # The system account means automated, and a person asked for + # this one. Null says a person we cannot name. + author = request.user if request.user.is_authenticated else None try: - service, created = import_from_url(url) + service, created = import_from_url(url, author=author) except ValueError as error: # `identify` raises this when no adapter could read the page. # It is the ordinary outcome of pasting the wrong address, diff --git a/api/polling/importer.py b/api/polling/importer.py index c8c4b19..7d7e964 100644 --- a/api/polling/importer.py +++ b/api/polling/importer.py @@ -22,19 +22,30 @@ from polling.models import PollRun from polling.reconcile import apply_fetch +# What `author` means when a caller passes nothing. A scheduled job +# and a management command sign their rows with the system account. +# `None` is a different answer: a person we cannot name. +AUTOMATED = object() -def import_from_url(url: str) -> tuple[Service, bool]: + +def import_from_url(url: str, author=AUTOMATED) -> tuple[Service, bool]: """Build a whole service from a status page URL. The provider, the name, the components and the event history all come from the page. Returns (service, created); importing a page that is already in the catalog returns the existing service. + `author` signs the service and its status page. A request path + passes the caller, so the admin who imported Twilio is on the row. + It falls back to the system account, which is true of a command. + Reading and writing are two steps, and only the write is a transaction. A provider takes seconds to answer, and a database connection must not spend them waiting on one. """ + if author is AUTOMATED: + author = get_user_model().objects.system() key = StatusPage.normalise_url(url) already = _imported(key) if already is not None: @@ -51,7 +62,7 @@ def import_from_url(url: str) -> tuple[Service, bool]: "source": adapter.status_source, "started": started, } - return _import(key, adapter_class, fetch_url, fetched) + return _import(key, adapter_class, fetch_url, fetched, author) def _imported(key): @@ -61,7 +72,7 @@ def _imported(key): @transaction.atomic -def _import(key, adapter_class, fetch_url, fetched): +def _import(key, adapter_class, fetch_url, fetched, author): """Write what the page said, in one transaction.""" # Somebody may have imported the same page while we read it. The # fetch is outside the transaction now, so that window is @@ -71,9 +82,6 @@ def _import(key, adapter_class, fetch_url, fetched): return already, False metadata = fetched["metadata"] - # Nobody typed these in, so they are signed by the system account - # rather than left blank. - author = get_user_model().objects.system() service = Service.objects.create( name=metadata["name"], homepage_url=metadata.get("homepage_url", ""), @@ -95,9 +103,8 @@ def _import(key, adapter_class, fetch_url, fetched): # the first reading has no provenance, and the log is missing # the request that made the rows. # - # Nothing above makes the poller. `create_poller` in - # polling.signals does, on the Service save. Making one here - # would trip the one-to-one column. + # Nothing above makes the poller. `Service.ensure_poller` does, on + # the save. Making one here would trip the one-to-one column. run = PollRun.open( service.poller, key, adapter_class.provider, at=fetched["started"] ) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index b9499dd..fdd70dd 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -184,7 +184,7 @@ def test_importing_from_the_admin_creates_the_service(staff_client, monkeypatch) monkeypatch.setattr( "catalog.admin.import_from_url", - lambda url: (Service.objects.create(name="Imported"), True), + lambda url, author=None: (Service.objects.create(name="Imported"), True), ) response = staff_client.post( "/admin/catalog/service/import-from-url/", @@ -194,11 +194,33 @@ def test_importing_from_the_admin_creates_the_service(staff_client, monkeypatch) assert Service.objects.filter(name="Imported").exists() +@pytest.mark.django_db +def test_importing_from_the_admin_records_the_admin_who_asked( + staff_client, monkeypatch +): + # The action has a request, so the row names the person. Dropping + # the caller signs the bot for work an admin did. + from catalog.models import Service + + def recording_import(url, author=None): + return Service.objects.create(name="Imported", created_by=author), True + + monkeypatch.setattr("catalog.admin.import_from_url", recording_import) + staff_client.post( + "/admin/catalog/service/import-from-url/", + {"status_page_url": "https://status.example.com/", "_form_submitted": "1"}, + ) + + assert Service.objects.get().created_by == User.objects.get( + email="admin@example.com" + ) + + @pytest.mark.django_db def test_an_unreadable_page_is_reported_not_a_500(staff_client, monkeypatch): # The URL came from a person pasting a stranger's page. Anything can # come back from it. - def boom(url): + def boom(url, author=None): raise ValueError("not a status page") monkeypatch.setattr("catalog.admin.import_from_url", boom) diff --git a/api/tests/test_import.py b/api/tests/test_import.py index b24edb8..274709b 100644 --- a/api/tests/test_import.py +++ b/api/tests/test_import.py @@ -234,3 +234,32 @@ def test_a_second_import_of_the_same_page_returns_the_first_service(): assert again is False assert second.pk == first.pk assert Service.objects.count() == 1 + + +@pytest.mark.django_db +def test_an_import_records_the_caller_who_asked_for_it(user): + # The bot signs automated rows only. Dropping the caller here + # records a person's import as the system account. + from tests.conftest import jwt_client + + jwt_client(user).post( + reverse("catalog-import"), + {"status_page_url": "https://status.twilio.com/"}, + format="json", + ) + + assert Service.objects.get().created_by == user + + +@pytest.mark.django_db +def test_an_anonymous_import_names_nobody_rather_than_the_bot(): + # The endpoint is AllowAny, so a signed-out person can import. A + # service request answers this the same way, with null. A person + # asked, and the system account would claim a job did. + APIClient().post( + reverse("catalog-import"), + {"status_page_url": "https://status.twilio.com/"}, + format="json", + ) + + assert Service.objects.get().created_by is None diff --git a/api/tests/test_polling.py b/api/tests/test_polling.py index 7fa9b7f..3613305 100644 --- a/api/tests/test_polling.py +++ b/api/tests/test_polling.py @@ -244,8 +244,25 @@ def fetch_service_metadata(self): @pytest.mark.django_db -def test_a_poller_nobody_added_is_signed_by_the_system(): - """The signal adds it, so the author says the system, not nobody. +def test_a_poller_carries_the_author_of_the_service_that_spawned_it(): + """A person caused the service, so the poller records that person. + + Stamping the system account here would name the bot for work a + person did. + """ + from catalog.models import Service + from tests.factories import UserFactory + + person = UserFactory() + + service = Service.objects.create(name="Owned", created_by=person) + + assert service.poller.created_by == person + + +@pytest.mark.django_db +def test_a_poller_for_a_service_with_no_author_is_signed_by_the_system(): + """The fallback, for a service nobody signed. A blank author reads the same as one that was lost. """ diff --git a/api/tests/test_service_requests.py b/api/tests/test_service_requests.py index b05e686..49af60d 100644 --- a/api/tests/test_service_requests.py +++ b/api/tests/test_service_requests.py @@ -100,7 +100,7 @@ def test_a_failed_import_does_not_spend_this_endpoints_budget(client, monkeypatc # shared scope would let the import spend this endpoint's budget. # An asker who had already tried six times could never send us # the URL that failed. - def refuses(url): + def refuses(url, author=None): raise ValueError(f"no adapter for {url}") monkeypatch.setattr("catalog.views.imports.import_from_url", refuses) From 905db5800524b99fb649fec0e4315117fcea46dd Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:28:18 -0700 Subject: [PATCH 62/94] feat: a service records how it reached the catalog The catalog holds services somebody typed in and services read off a status page. Nothing said which. `created_by` names the person, and a person is on both paths, so it cannot answer this. `source` answers it, with two values, because there are two writers. `import_from_url` sets import, and everything else reads manual, which is true of a row nobody imported. It stays out of the API. No screen in the client spec reads it, and the contract is what the client is generated from. The admin lists and filters on it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 2 + api/catalog/choices.py | 12 ++++++ api/catalog/migrations/0012_service_source.py | 40 +++++++++++++++++++ api/catalog/models.py | 10 ++++- api/polling/importer.py | 2 + api/tests/test_admin.py | 15 +++++++ api/tests/test_import.py | 15 +++++++ docs/specs/2026-08-23-statusboard-design.md | 14 ++++--- 8 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 api/catalog/migrations/0012_service_source.py diff --git a/api/catalog/admin.py b/api/catalog/admin.py index 210e808..2fc1b90 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -315,6 +315,7 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): "display_service", "display_severity", "provider", + "source", "display_related", ] search_fields = [ @@ -329,6 +330,7 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): list_filter = [ ServiceSeverityFilter, ("status_page__provider", ChoicesDropdownFilter), + ("source", ChoicesDropdownFilter), TrackedFilter, ("created_at", RangeDateTimeFilter), ("updated_at", RangeDateTimeFilter), diff --git a/api/catalog/choices.py b/api/catalog/choices.py index 71c4010..cc992d3 100644 --- a/api/catalog/choices.py +++ b/api/catalog/choices.py @@ -23,3 +23,15 @@ class StatusPageProvider(models.TextChoices): ORACLE = "oracle", "Oracle" GOOGLE_FEED = "google_feed", "Google feed" RSS = "rss", "RSS" + + +class ServiceSource(models.TextChoices): + """How a service reached the catalog. + + Two values, because there are two writers. `import_from_url` makes + one, and the admin's add form makes the other. Anything else, such + as a shell or a migration, reads manual. It was not imported. + """ + + MANUAL = "manual", "Added by hand" + IMPORT = "import", "Imported from a status page" diff --git a/api/catalog/migrations/0012_service_source.py b/api/catalog/migrations/0012_service_source.py new file mode 100644 index 0000000..165029f --- /dev/null +++ b/api/catalog/migrations/0012_service_source.py @@ -0,0 +1,40 @@ +from django.db import migrations, models + +# How a service reached the catalog. `created_by` says who added it, and +# this says how. Every existing row reads manual, which is true of one +# nobody imported. + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0011_drop_derived_ancestry_and_search"), + ] + + operations = [ + migrations.AddField( + model_name="historicalservice", + name="source", + field=models.CharField( + choices=[ + ("manual", "Added by hand"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + migrations.AddField( + model_name="service", + name="source", + field=models.CharField( + choices=[ + ("manual", "Added by hand"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + ] diff --git a/api/catalog/models.py b/api/catalog/models.py index 94bc64d..d58ef3a 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -6,7 +6,7 @@ from django.utils.text import slugify from simple_history.models import HistoricalRecords -from catalog.choices import StatusPageProvider +from catalog.choices import ServiceSource, StatusPageProvider from common.models import BaseModel from common.queries import related_count @@ -56,6 +56,14 @@ class Service(BaseModel): ) logo = models.URLField(blank=True, default="") homepage_url = models.URLField(blank=True, default="") + # How the row arrived. `created_by` says who, and this says how. + # Admin only: no screen reads it, so no serializer carries it. + source = models.CharField( + max_length=32, + choices=ServiceSource.choices, + default=ServiceSource.MANUAL, + db_default=ServiceSource.MANUAL, + ) objects = ServiceQuerySet.as_manager() history = HistoricalRecords() diff --git a/api/polling/importer.py b/api/polling/importer.py index 7d7e964..545e85d 100644 --- a/api/polling/importer.py +++ b/api/polling/importer.py @@ -17,6 +17,7 @@ from django.db import transaction from django.utils import timezone +from catalog.choices import ServiceSource from catalog.models import Service, StatusPage from polling.adapters.registry import identify from polling.models import PollRun @@ -86,6 +87,7 @@ def _import(key, adapter_class, fetch_url, fetched, author): name=metadata["name"], homepage_url=metadata.get("homepage_url", ""), logo=fetched["logo"], + source=ServiceSource.IMPORT, created_by=author, updated_by=author, ) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index fdd70dd..b970c51 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -161,6 +161,21 @@ def test_the_admin_stamps_who_created_a_row(staff_client): assert service.updated_by.email == "admin@example.com" +@pytest.mark.django_db +def test_a_service_added_on_the_admin_form_says_it_was_added_by_hand(staff_client): + # The add form is the one path that is not an import. A row saying + # import here claims a status page named the service. + from catalog.choices import ServiceSource + from catalog.models import Service + + staff_client.post( + reverse("admin:catalog_service_add"), + service_form_data(staff_client, name="By hand"), + ) + + assert Service.objects.get(name="By hand").source == ServiceSource.MANUAL + + @pytest.mark.django_db @pytest.mark.parametrize("field", ["created_by", "updated_by"]) def test_derived_fields_are_not_on_the_form(staff_client, field): diff --git a/api/tests/test_import.py b/api/tests/test_import.py index 274709b..a4d5d5b 100644 --- a/api/tests/test_import.py +++ b/api/tests/test_import.py @@ -263,3 +263,18 @@ def test_an_anonymous_import_names_nobody_rather_than_the_bot(): ) assert Service.objects.get().created_by is None + + +@pytest.mark.django_db +def test_an_imported_service_says_it_was_imported(): + # The human tells a service added by hand from one read off a page. + # A row left at the default claims somebody typed it in. + from catalog.choices import ServiceSource + + APIClient().post( + reverse("catalog-import"), + {"status_page_url": "https://status.twilio.com/"}, + format="json", + ) + + assert Service.objects.get().source == ServiceSource.IMPORT diff --git a/docs/specs/2026-08-23-statusboard-design.md b/docs/specs/2026-08-23-statusboard-design.md index 125b947..b8ae3dc 100644 --- a/docs/specs/2026-08-23-statusboard-design.md +++ b/docs/specs/2026-08-23-statusboard-design.md @@ -128,6 +128,7 @@ erDiagram string name url logo url homepage_url + enum source "TextChoices, admin only, never served" } StatusPage { uuid id PK @@ -257,16 +258,19 @@ is a different concept from a permission role. ### catalog -- `Service` — slug, name, logo, homepage URL +- `Service` — slug, name, logo, homepage URL, source `logo` comes from the provider's page and is refreshed on every poll, so a rebrand upstream does not leave a stale entry in the catalogue. + `source` says how the row arrived: `manual` from the admin's add form, `import` from + `import_from_url`. It is admin only. No screen reads it, so no serializer carries it. + Three earlier fields are gone. `added_by` duplicated `BaseModel.created_by`, which every model - already carries. `is_curated` had no consumer: nothing sorted, filtered or rendered by it. - "Seeded by us rather than pasted by someone" is `created_by IS NULL`. `watcher_count` was a - column a signal kept true, and four write paths never reached the signal. It is counted when it - is read instead. + already carries. `created_by` says who added a service and `source` says how. Neither is null + for a service a signed-in person added. `is_curated` had no consumer: nothing sorted, filtered + or rendered by it. `watcher_count` was a column a signal kept true, and four write paths never + reached the signal. It is counted when it is read instead. - `StatusPage` — **its own model**, `OneToOneField` to `Service`. `url` (normalised, **unique**), `provider`, `api_url_override` (blank unless needed). What and where — nothing about polling. - `Poller` — `OneToOneField` to **`Service`**, created with it. The thing that reads a service: From 451dc8dc00105ce295b31a621aa01a27d74f663a Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:47:06 -0700 Subject: [PATCH 63/94] fix: service requests join the catalog tabs, not the sidebar A sidebar link put service requests one level away from the catalog they describe. A request names something the catalog lacks, so it belongs with the services, components, events and status history it sits beside, the way UNFOLD["TABS"] already groups a service's own parts. Tabs also read active from any page in the group, so a person already on the catalog now finds it without leaving. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/settings.py | 16 +++++++++------- api/tests/test_admin.py | 32 ++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/api/api/settings.py b/api/api/settings.py index 3152fe6..79b9dc1 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -277,8 +277,10 @@ "COMMAND": {"search_models": True, "show_history": True}, # The sidebar is two levels deep. It cannot nest an item under an # item, so tabs carry the relationship. A component belongs to a - # service, and a status history to a component. All four are one - # page. The sidebar entry stays active on any of its tabs. + # service, and a status history to a component. A service request + # names something the catalog lacks, so it lives beside the catalog + # too. All five are one page. The sidebar entry stays active on any + # of its tabs. "TABS": [ { "models": [ @@ -286,6 +288,7 @@ "catalog.servicecomponent", "status.serviceevent", "status.componentstatus", + "catalog.servicerequest", ], "items": [ { @@ -304,6 +307,10 @@ "title": _("Status history"), "link": reverse_lazy("admin:status_componentstatus_changelist"), }, + { + "title": _("Service requests"), + "link": reverse_lazy("admin:catalog_servicerequest_changelist"), + }, ], }, { @@ -367,11 +374,6 @@ "icon": "person", "link": reverse_lazy("admin:authentication_user_changelist"), }, - { - "title": _("Service requests"), - "icon": "playlist_add", - "link": reverse_lazy("admin:catalog_servicerequest_changelist"), - }, ], }, ], diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index b970c51..54ccdf2 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -709,17 +709,33 @@ def test_the_sidebar_names_every_top_level_destination(): for group in settings.UNFOLD["SIDEBAR"]["navigation"] for item in group["items"] } - for title in ( - "Dashboard", - "Services", - "Polling", - "Boards", - "Users", - "Service requests", - ): + for title in ("Dashboard", "Services", "Polling", "Boards", "Users"): assert title in titles, f"{title} has no sidebar entry" +def _tab_strip(body): + """The tab strip's own markup, not the whole page. + + The sidebar links to a service or a poller too, on every admin + page. Searching the full body would pass on that link alone and + never prove a tab exists. + """ + start = body.index('id="tabs-items"') + return body[start : body.index("", start)] + + +@pytest.mark.django_db +def test_the_catalog_tabs_carry_service_requests(staff_client): + # A sidebar link once pointed here on its own. A thing that belongs + # to the catalog belongs on the catalog's tabs instead. + body = staff_client.get( + reverse("admin:catalog_service_changelist") + ).content.decode() + tabs = _tab_strip(body) + assert 'href="/admin/catalog/servicerequest/"' in tabs + assert "Service requests" in tabs + + @pytest.mark.django_db def test_the_component_list_offers_the_featured_checkbox(admin_client, db): # list_editable renders the flag as a checkbox in the row. Without From 4ff22a49b8e63c16947ce5b7b9b417973336a774 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:47:45 -0700 Subject: [PATCH 64/94] fix: give magic link tokens a tab beside their user MagicLinkTokenAdmin answered but nothing in the admin pointed at it: no sidebar entry, no tab, no inline. A token belongs to a user the way a component belongs to a service, so it gets the same treatment as the catalog and polling groups: a tab on the page of the thing it belongs to, rather than a route of its own. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/settings.py | 20 ++++++++++++++++++++ api/tests/test_admin.py | 12 ++++++++++++ 2 files changed, 32 insertions(+) diff --git a/api/api/settings.py b/api/api/settings.py index 79b9dc1..775587a 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -338,6 +338,26 @@ }, ], }, + { + # A magic link belongs to a user. It has no reason to be + # findable on its own. + "models": [ + "authentication.user", + "authentication.magiclinktoken", + ], + "items": [ + { + "title": _("Users"), + "link": reverse_lazy("admin:authentication_user_changelist"), + }, + { + "title": _("Magic links"), + "link": reverse_lazy( + "admin:authentication_magiclinktoken_changelist" + ), + }, + ], + }, ], "SIDEBAR": { "show_search": True, diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 54ccdf2..ce3468b 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -736,6 +736,18 @@ def test_the_catalog_tabs_carry_service_requests(staff_client): assert "Service requests" in tabs +@pytest.mark.django_db +def test_the_user_tabs_carry_magic_links(staff_client): + # MagicLinkTokenAdmin answered but nothing pointed at it. A token + # belongs to a user, so its tab sits on the user's page. + body = staff_client.get( + reverse("admin:authentication_user_changelist") + ).content.decode() + tabs = _tab_strip(body) + assert 'href="/admin/authentication/magiclinktoken/"' in tabs + assert "Magic links" in tabs + + @pytest.mark.django_db def test_the_component_list_offers_the_featured_checkbox(admin_client, db): # list_editable renders the flag as a checkbox in the row. Without From 990ca6c962db025acb775d23e78d1610a80716d4 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:48:30 -0700 Subject: [PATCH 65/94] fix: rename the catalog sidebar entry from Services to Catalog The entry now activates for five models across catalog and status, not the four it already covered. "Services" undersold that and read oddly from the events or service requests tab. "Catalog" is not literally exact either, since two of the five are status models, but it names the group closer than the entry it replaces. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/api/settings.py | 4 +++- api/tests/test_admin.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/api/api/settings.py b/api/api/settings.py index 775587a..1ad31cd 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -375,7 +375,9 @@ "link": reverse_lazy("admin:index"), }, { - "title": _("Services"), + # Five models across catalog and status answer + # here now, not only the services themselves. + "title": _("Catalog"), "icon": "lan", "link": reverse_lazy("admin:catalog_service_changelist"), }, diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index ce3468b..9d6b038 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -709,7 +709,7 @@ def test_the_sidebar_names_every_top_level_destination(): for group in settings.UNFOLD["SIDEBAR"]["navigation"] for item in group["items"] } - for title in ("Dashboard", "Services", "Polling", "Boards", "Users"): + for title in ("Dashboard", "Catalog", "Polling", "Boards", "Users"): assert title in titles, f"{title} has no sidebar entry" @@ -748,6 +748,15 @@ def test_the_user_tabs_carry_magic_links(staff_client): assert "Magic links" in tabs +@pytest.mark.django_db +def test_the_sidebar_entry_is_named_for_the_whole_group(staff_client): + # Five models across catalog and status answer here now, not just + # the services themselves. "Services" read oddly from the events + # tab and undersold what the entry opens. + body = staff_client.get(reverse("admin:index")).content.decode() + assert "Catalog" in body + + @pytest.mark.django_db def test_the_component_list_offers_the_featured_checkbox(admin_client, db): # list_editable renders the flag as a checkbox in the row. Without From fbc9981e2cfd322ce2ef8a02cec3a60c724ae20b Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:49:31 -0700 Subject: [PATCH 66/94] test: derive admin reachability from the registry, not a hand list The hardcoded sidebar-titles test could only ever check what somebody remembered to add to it. That is the same shape that let ServiceRequest and MagicLinkToken ship unreachable in the first place, and it cannot tell "reachable through a parent" from "reachable by nothing". The replacement reads the admin registry directly. A model counts as reachable when the sidebar or a tab group names it, or when it is the model of an inline declared on an admin already reachable that way. Running it against the admin as it stood before this branch's other commits fails on exactly one label, authentication.MagicLinkToken, which is the gap a human went looking for by eye. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_admin.py | 93 +++++++++++++++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 12 deletions(-) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 9d6b038..3c94e6a 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -1,10 +1,77 @@ +import re + import pytest +from django.conf import settings from django.contrib import admin from django.urls import reverse from django.utils import timezone from authentication.models import User +# The apps this project owns. A third-party admin, such as +# auth.Group or a simplejwt token table, is not ours to place. +# It sits outside what this test requires reachable. +OUR_APPS = {"catalog", "status", "authentication", "dashboards", "polling"} + +CHANGELIST_PATH = re.compile(r"^/admin/(?P[^/]+)/(?P[^/]+)/$") + + +def _changelist_target(link): + """The registered model a changelist link opens, or None. + + A link to something other than a changelist, such as the + dashboard, names no model. It is not a gap to report. + """ + match = CHANGELIST_PATH.match(str(link)) + if not match: + return None + app_label, model_name = match.group("app_label"), match.group("model_name") + for model in admin.site._registry: + if model._meta.app_label == app_label and model._meta.model_name == model_name: + return model._meta.label + return None + + +def _sidebar_reachable(): + return { + label + for group in settings.UNFOLD["SIDEBAR"]["navigation"] + for item in group["items"] + if (label := _changelist_target(item["link"])) + } + + +def _tab_reachable(): + labels = set() + for group in settings.UNFOLD["TABS"]: + for dotted in group["models"]: + app_label, model_name = dotted.split(".") + for model in admin.site._registry: + if ( + model._meta.app_label == app_label + and model._meta.model_name == model_name + ): + labels.add(model._meta.label) + return labels + + +def _inline_reachable(top_level): + """A model reachable only as an inline on an already-reachable page. + + An inline on a page nothing else reaches does not count. Reachable + describes a page a person can open, not a class attribute nobody + links to. + """ + labels = set() + for model, site_admin in admin.site._registry.items(): + if model._meta.label not in top_level: + continue + for inline in getattr(site_admin, "inlines", []): + inline_model = getattr(inline, "model", None) + if inline_model is not None: + labels.add(inline_model._meta.label) + return labels + @pytest.fixture def staff_client(client): @@ -698,19 +765,21 @@ def test_service_requests_are_listed_by_demand(admin_client, db): assert response.content.index(b"b.example") < response.content.index(b"a.example") -def test_the_sidebar_names_every_top_level_destination(): - # A changelist that answers is not one anyone can find. The sidebar - # is the only listing browsed without a URL. Every top-level page - # needs a title in it. - from django.conf import settings - - titles = { - str(item["title"]) - for group in settings.UNFOLD["SIDEBAR"]["navigation"] - for item in group["items"] +def test_every_registered_admin_is_reachable(): + # A registered admin nothing links to is a page nobody finds, the + # way magic links once shipped. Reachable means named in the + # sidebar, named in a tab group, or shown as a parent's inline. + # A hardcoded title list cannot tell that apart from an admin + # reachable by nothing at all. + registered = { + model._meta.label + for model in admin.site._registry + if model._meta.app_label in OUR_APPS } - for title in ("Dashboard", "Catalog", "Polling", "Boards", "Users"): - assert title in titles, f"{title} has no sidebar entry" + top_level = _sidebar_reachable() | _tab_reachable() + reachable = top_level | _inline_reachable(top_level) + missing = registered - reachable + assert not missing, f"registered but unreachable: {sorted(missing)}" def _tab_strip(body): From 9c0892ef316d9d94592019045294f4cad7f5fe48 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 15:59:47 -0700 Subject: [PATCH 67/94] admin: move featured flag and archive fields in component form Featured is a thing an admin sets on a component. Archiving is a state that polling reports. They should not sit together on the form; the featured flag belongs with basic properties, and archive state belongs in its own section. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 5 ++--- api/tests/test_admin.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index 2fc1b90..bb5ffca 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -554,8 +554,7 @@ class ServiceComponentAdmin( "service", "name", "external_id", - "is_archived", - "archived_at", + "is_featured", ] }, ), @@ -563,7 +562,7 @@ class ServiceComponentAdmin( _("Position"), {"fields": ["parent", "status_page_order", "is_overall"]}, ), - (_("Featuring"), {"fields": ["is_featured"]}), + (_("Archive"), {"fields": ["is_archived", "archived_at"]}), audit_section(), ] inlines = [ComponentStatusInline] diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 3c94e6a..a145939 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -1177,3 +1177,41 @@ def test_the_components_link_opens_exactly_what_it_counts(admin_client): link = related_item(admin_user, service, "Components")["link"] assert admin_client.get(link).context["cl"].result_count == 1 + + +@pytest.mark.django_db +def test_component_admin_fieldsets_order_featured_and_archive(): + # Featured moved to the base section. Archive fields moved to their + # own section. Verify the fieldsets layout against regressions. + from catalog.models import ServiceComponent + + model_admin = admin.site._registry[ServiceComponent] + fieldsets = model_admin.get_fieldsets(None) + + # Four sections: base, Position, Archive, audit. + assert len(fieldsets) == 4, f"Expected 4 fieldsets, got {len(fieldsets)}" + + # First section (base): has is_featured, not archive fields. + base_section = fieldsets[0] + assert base_section[0] is None, "First section should have no title" + base_fields = base_section[1]["fields"] + assert "is_featured" in base_fields, "is_featured must be in base section" + assert "is_archived" not in base_fields, "is_archived must not be in base section" + assert "archived_at" not in base_fields, "archived_at must not be in base section" + + # Second section: Position (unchanged). + position_section = fieldsets[1] + assert "Position" in str(position_section[0]), "Second section must be Position" + assert "parent" in position_section[1]["fields"], ( + "Position section must have parent" + ) + + # Third section: Archive with archive fields. + archive_section = fieldsets[2] + assert "Archive" in str(archive_section[0]), "Third section must be Archive" + archive_fields = archive_section[1]["fields"] + assert "is_archived" in archive_fields, "is_archived must be in Archive section" + assert "archived_at" in archive_fields, "archived_at must be in Archive section" + + # Fourth section: audit (checked by presence). + assert fieldsets[3] is not None, "Fourth section (audit) must exist" From ca3952a94ef58c64e4755bc1120ab80a30a5fd70 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:02:23 -0700 Subject: [PATCH 68/94] fix: an ordering value the list does not declare answers 400 `?fields=` refused an unknown name and `?ordering=` dropped it, then sorted by the default. A caller read a list in an order it did not ask for, with nothing in the response to say so. `-suggested` was worse. `suggested` is an editorial order that names its own directions, so the prefix built `--is_featured` and the ORM raised from there. A hand written caller reached a 500. Both refusals live in the one ordering backend, and the contract carries the 400 on every list that takes the parameter. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/views/components.py | 12 +++++- api/common/filters.py | 8 ++++ api/common/ordering.py | 21 ++++++++-- api/dashboards/views.py | 10 ++++- api/status/views.py | 12 +++++- api/tests/test_ordering.py | 68 +++++++++++++++++++++++++++++++-- docs/api/openapi.yaml | 19 +++++++++ 7 files changed, 136 insertions(+), 14 deletions(-) diff --git a/api/catalog/views/components.py b/api/catalog/views/components.py index f98a3e4..bd0fb77 100644 --- a/api/catalog/views/components.py +++ b/api/catalog/views/components.py @@ -6,7 +6,7 @@ """ from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view from rest_framework import generics from rest_framework.permissions import AllowAny @@ -15,7 +15,7 @@ from catalog.queries import COMPONENT_WATCHER_COUNT from catalog.serializers import ComponentSerializer from common.aggregates import StatusAggregateSet -from common.filters import FieldsBackend +from common.filters import REJECTED_PARAMETER, FieldsBackend from common.ordering import MappedOrderingFilter from common.serializers import ErrorSerializer from status.queries import CURRENT_SEVERITY @@ -40,6 +40,14 @@ def get_queryset(self): ) +@extend_schema_view( + get=extend_schema( + responses={ + 200: ComponentSerializer, + 400: OpenApiResponse(description=REJECTED_PARAMETER), + } + ) +) class ComponentListView(ComponentQueryMixin, generics.ListAPIView): aggregate_set = StatusAggregateSet filterset_class = ComponentFilter diff --git a/api/common/filters.py b/api/common/filters.py index 8cad26a..c349282 100644 --- a/api/common/filters.py +++ b/api/common/filters.py @@ -6,6 +6,14 @@ "nested object rather than dropping it." ) +# One description for the one 400 a list answers. `fields`, `ordering` +# and every filter refuse the same way, so a client branches once. +REJECTED_PARAMETER = ( + "A query parameter names something this list does not serve, or " + "carries a value it cannot read. The body is a field-errors object, " + "keyed by the parameter." +) + class FieldsBackend(BaseFilterBackend): """Declare `?fields=` for drf-spectacular. diff --git a/api/common/ordering.py b/api/common/ordering.py index 78d3fbb..5283d23 100644 --- a/api/common/ordering.py +++ b/api/common/ordering.py @@ -5,6 +5,7 @@ importing `catalog` and `status` back. """ +from rest_framework.exceptions import ValidationError from rest_framework.filters import OrderingFilter @@ -13,6 +14,10 @@ class MappedOrderingFilter(OrderingFilter): A cursor cannot order on a related path or on a name that is not a field. A view declares `ordering_map` and annotates the flat field it names. + + A value the view does not declare answers 400, the way `?fields=` + refuses an unknown name. Dropping it left the caller reading a list + sorted by something else, with nothing to say so. """ def remove_invalid_fields(self, queryset, fields, view, request): @@ -20,9 +25,17 @@ def remove_invalid_fields(self, queryset, fields, view, request): allowed = set(getattr(view, "ordering_fields", ())) | set(mapping) out = [] for term in fields: - name = term.lstrip("-") + name = term.removeprefix("-") if name not in allowed: - continue - for mapped in mapping.get(name, [name]): - out.append(f"-{mapped}" if term.startswith("-") else mapped) + raise ValidationError({"ordering": [f"Unknown ordering: {term}."]}) + mapped = mapping.get(name, [name]) + if not term.startswith("-"): + out.extend(mapped) + elif len(mapped) == 1 and not mapped[0].startswith("-"): + out.append(f"-{mapped[0]}") + else: + # An editorial order already names its own directions. + # A second prefix built `--is_featured`, which reached + # the ORM as a 500. + raise ValidationError({"ordering": [f"Cannot reverse: {term}."]}) return out diff --git a/api/dashboards/views.py b/api/dashboards/views.py index 42ac427..5e11736 100644 --- a/api/dashboards/views.py +++ b/api/dashboards/views.py @@ -9,7 +9,7 @@ from catalog.models import ServiceComponent from catalog.serializers import ComponentSerializer from common.aggregates import StatusAggregateSet -from common.filters import FieldsBackend +from common.filters import REJECTED_PARAMETER, FieldsBackend from common.ordering import MappedOrderingFilter from dashboards.filters import BoardComponentFilter from dashboards.models import Dashboard, DashboardItem @@ -26,6 +26,12 @@ def _board(request, uuid): # dispatches to. An annotation on `create` was dropped in silence, so # the documented 200 and 404 reached no schema. @extend_schema_view( + get=extend_schema( + responses={ + 200: ComponentSerializer, + 400: OpenApiResponse(description=REJECTED_PARAMETER), + } + ), post=extend_schema( request=TrackComponentSerializer, responses={ @@ -36,7 +42,7 @@ def _board(request, uuid): ), 404: OpenApiResponse(description="No such board, or no such component."), }, - ) + ), ) class BoardComponentListView(generics.ListCreateAPIView): # Schema generation reads the model off this before it calls diff --git a/api/status/views.py b/api/status/views.py index acd8d49..4d27628 100644 --- a/api/status/views.py +++ b/api/status/views.py @@ -9,13 +9,13 @@ from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view from rest_framework import generics from rest_framework.exceptions import NotAuthenticated from rest_framework.permissions import AllowAny from common.aggregates import EventAggregateSet -from common.filters import FieldsBackend +from common.filters import REJECTED_PARAMETER, FieldsBackend from common.ordering import MappedOrderingFilter from common.pagination import TimelinePagination from common.serializers import ErrorSerializer @@ -29,6 +29,14 @@ ) +@extend_schema_view( + get=extend_schema( + responses={ + 200: ServiceEventSerializer, + 400: OpenApiResponse(description=REJECTED_PARAMETER), + } + ) +) class EventListView(generics.ListAPIView): permission_classes = [AllowAny] serializer_class = ServiceEventSerializer diff --git a/api/tests/test_ordering.py b/api/tests/test_ordering.py index d168e49..1aff1e0 100644 --- a/api/tests/test_ordering.py +++ b/api/tests/test_ordering.py @@ -1,4 +1,9 @@ +import pytest +from django.urls import reverse +from rest_framework.exceptions import ValidationError + from common.ordering import MappedOrderingFilter +from tests.factories import ComponentFactory, ServiceFactory, track def _view(ordering_fields, ordering_map): @@ -17,12 +22,23 @@ def test_a_related_path_ordering_does_not_reach_the_paginator(): assert all("__" not in term for term in out) -def test_an_unmapped_unknown_value_is_dropped(): +def test_an_unmapped_unknown_value_is_refused(): + # Dropping it sorted the list by the default instead. The caller + # asked for one order and read another, with nothing to say so. view = _view(["name"], {}) - assert ( + with pytest.raises(ValidationError) as raised: MappedOrderingFilter().remove_invalid_fields(None, ["nonsense"], view, None) - == [] - ) + assert raised.value.detail == {"ordering": ["Unknown ordering: nonsense."]} + + +def test_reversing_a_key_that_names_its_own_directions_is_refused(): + # `-suggested` prefixed every mapped term and built `--is_featured`. + # The ORM raised FieldError from there, which reached the caller as + # a 500. + view = _view(["name"], {"suggested": ["-is_featured", "name"]}) + with pytest.raises(ValidationError) as raised: + MappedOrderingFilter().remove_invalid_fields(None, ["-suggested"], view, None) + assert raised.value.detail == {"ordering": ["Cannot reverse: -suggested."]} def test_a_value_that_is_not_a_field_expands_to_several(): @@ -37,3 +53,47 @@ def test_a_plain_declared_field_passes_through(): assert MappedOrderingFilter().remove_invalid_fields( None, ["-name"], view, None ) == ["-name"] + + +def _component_list(client, authed, board): + ComponentFactory(service=ServiceFactory(), is_overall=True) + return client, reverse("component-list") + + +def _event_list(client, authed, board): + return client, reverse("event-list") + + +def _board_components(client, authed, board): + track(ComponentFactory(), user=board.owner) + return authed, reverse("board-components", kwargs={"uuid": board.id}) + + +# One builder per operation the contract gives an `ordering` parameter. +# A list added later and left out here would shrug again. +ORDERING_PROBES = { + "/catalog/components/": _component_list, + "/events/": _event_list, + "/dashboards/{uuid}/components/": _board_components, +} + + +@pytest.mark.django_db +@pytest.mark.parametrize("path", sorted(ORDERING_PROBES)) +def test_an_unknown_ordering_is_refused(path, client, authenticated_client, board): + # `?fields=` refuses an unknown name and `?ordering=` shrugged. One + # parameter that answers 400 and one that does not is two contracts. + caller, url = ORDERING_PROBES[path](client, authenticated_client, board) + response = caller.get(url, {"ordering": "nonsense"}) + assert response.status_code == 400, f"GET {path} answered {response.status_code}" + assert response.json() == {"ordering": ["Unknown ordering: nonsense."]} + + +@pytest.mark.django_db +def test_reversing_suggested_is_a_400_not_a_500(client, authenticated_client, board): + # `suggested` is four terms, two of them already descending. The + # prefix built `--is_featured` and the ORM raised on it. + caller, url = _component_list(client, authenticated_client, board) + response = caller.get(url, {"ordering": "-suggested"}) + assert response.status_code == 400 + assert response.json() == {"ordering": ["Cannot reverse: -suggested."]} diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 271f0dc..b28995c 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -119,6 +119,9 @@ info: Lists return `{ aggregates, next, results }`. `aggregates` covers the whole collection, not the page. `ordering` follows DRF. Prefix a field with `-` to reverse it. GETs are ETagged. + An `ordering` name the list does not declare answers `400`, as an unknown `fields` name + does. `suggested` names its own directions, so `-suggested` answers `400` as well. + ## Identity Every resource has a UUID. `Service` also has a stable `slug`. Public catalog routes use the @@ -339,6 +342,7 @@ paths: properties: aggregates: { $ref: '#/components/schemas/StatusAggregates' } results: { type: array, items: { $ref: '#/components/schemas/Component' } } + '400': { $ref: '#/components/responses/RejectedParameter' } /catalog/components/{uuid}/: get: @@ -400,6 +404,7 @@ paths: properties: aggregates: { $ref: '#/components/schemas/EventAggregates' } results: { type: array, items: { $ref: '#/components/schemas/ServiceEvent' } } + '400': { $ref: '#/components/responses/RejectedParameter' } /events/{uuid}/: get: @@ -473,6 +478,7 @@ paths: properties: aggregates: { $ref: '#/components/schemas/StatusAggregates' } results: { type: array, items: { $ref: '#/components/schemas/Component' } } + '400': { $ref: '#/components/responses/RejectedParameter' } post: tags: [board] summary: Track a component @@ -535,6 +541,19 @@ components: Error: description: Error content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } + RejectedParameter: + description: | + A query parameter names something the list does not serve, or carries a value it + cannot read. `fields`, `ordering` and every filter refuse the same way, so a client + branches once. + + The body is a field-errors object, one array of messages keyed by the parameter + name. It is not the `Error` shape. + content: + application/json: + schema: + type: object + additionalProperties: { type: array, items: { type: string } } schemas: Error: type: object From 426735b93b71301a968cbaba6b414ed2f73c6274 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:04:28 -0700 Subject: [PATCH 69/94] refactor: the dashboard filter decides who owns the board `dashboard` was owned twice. The view read the raw parameter, parsed the UUID by hand and checked the owner, while the filter did the narrowing. Two readers of one parameter drift. The check moves into `filter_dashboard`. django-filter validates the value before calling it, so the hand parse and its try/except go. A malformed id still answers 400, a stranger's board still answers 404, and an anonymous caller still answers 401. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/status/filters.py | 17 +++++++++++++++-- api/status/views.py | 32 +++----------------------------- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/api/status/filters.py b/api/status/filters.py index af40cbb..ec5646f 100644 --- a/api/status/filters.py +++ b/api/status/filters.py @@ -4,8 +4,11 @@ and it is read on its own more often than the view around it. """ +from django.shortcuts import get_object_or_404 from django_filters import rest_framework as filters +from rest_framework.exceptions import NotAuthenticated +from dashboards.models import Dashboard from status.choices import CLOSED_PHASES, EventPhaseState from status.models import ServiceEvent @@ -32,9 +35,19 @@ def filter_dashboard(self, queryset, name, value): """Everything posted across the services on one board. The board's rows are components, and an event names components. - A user reaches only their own board: the view checks the owner - before this runs. + + Who owns the board is decided here too, because one parameter + belongs to one place. The view read the id again and parsed the + UUID by hand. django-filter has parsed it before this runs, and + a malformed one never arrives. """ + # An anonymous caller owns no board. Refuse before comparing + # against AnonymousUser, which get_object_or_404 cannot do. + if not self.request.user.is_authenticated: + raise NotAuthenticated + # 404 rather than 403: someone else's board id should not be + # confirmable. + get_object_or_404(Dashboard, id=value, owner=self.request.user) return queryset.filter( affected_components__tracked_by__dashboard_id=value ).distinct() diff --git a/api/status/views.py b/api/status/views.py index 4d27628..3e8d8ce 100644 --- a/api/status/views.py +++ b/api/status/views.py @@ -5,13 +5,10 @@ publishes the labels. """ -import uuid - from django.shortcuts import get_object_or_404 from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view from rest_framework import generics -from rest_framework.exceptions import NotAuthenticated from rest_framework.permissions import AllowAny from common.aggregates import EventAggregateSet @@ -19,7 +16,6 @@ from common.ordering import MappedOrderingFilter from common.pagination import TimelinePagination from common.serializers import ErrorSerializer -from dashboards.models import Dashboard from status.filters import EventFilter from status.models import EventUpdate, ServiceEvent from status.serializers import ( @@ -46,31 +42,9 @@ class EventListView(generics.ListAPIView): ordering_fields = ["starts_at", "ends_at"] ordering_map = {} ordering = ["-starts_at"] - # get_queryset reads request.query_params, which schema generation - # has not set. This names the model without running it. - queryset = ServiceEvent.objects.none() - - def get_queryset(self): - queryset = ServiceEvent.objects.select_related("service") - board = self.request.query_params.get("dashboard") - if not board: - return queryset - try: - uuid.UUID(board) - except ValueError: - # Not a UUID. `EventFilter`'s own `dashboard` field rejects - # it next, in the shape any other malformed filter value - # gets. This method only owns whether the board is yours. - return queryset - # An anonymous caller cannot own a board. Refuse before - # comparing against AnonymousUser, which get_object_or_404 - # cannot do. - if not self.request.user.is_authenticated: - raise NotAuthenticated - # 404 rather than 403: someone else's board id should not - # be confirmable. The filter runs after this. - get_object_or_404(Dashboard, id=board, owner=self.request.user) - return queryset + # `EventFilter.filter_dashboard` narrows this to one board, and + # decides whether the caller owns it. + queryset = ServiceEvent.objects.select_related("service") @extend_schema_view( From f8bbeca76799acf39a8e64a298109075b1fc71d7 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:14:11 -0700 Subject: [PATCH 70/94] refactor: q is a declared filter beside service and ancestor The view read `q` out of the request and applied the search outside the filter set. `q` is a documented parameter sitting next to `service`, `ancestor` and `event`, which are all declared. A check belongs in the mechanism that already does that job. Declaring it puts every narrowing of the component list in one place, and the generated schema now carries the parameter the contract always named. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/filters.py | 10 +++++++++ api/catalog/views/components.py | 15 ++++--------- api/tests/test_admin.py | 38 --------------------------------- 3 files changed, 14 insertions(+), 49 deletions(-) diff --git a/api/catalog/filters.py b/api/catalog/filters.py index 6f4a480..4f71c7e 100644 --- a/api/catalog/filters.py +++ b/api/catalog/filters.py @@ -13,6 +13,7 @@ class ComponentFilter(SeverityFilterMixin): + q = filters.CharFilter(method="filter_q") service = filters.CharFilter(field_name="service__slug") # Every descendant, not one level. `parent` would name a query this # does not run. @@ -26,6 +27,15 @@ class Meta: model = ServiceComponent fields = {"is_overall": ["exact"]} + def filter_q(self, queryset, name, value): + """Narrow to what the caller typed, beside the other parameters. + + `ServiceComponentQuerySet.search` is the one definition of what a + word matches. The view read `q` out of the request instead. A + documented parameter then sat outside the filter set. + """ + return queryset.search(value) + def filter_ancestor(self, queryset, name, value): """Ask the tree for the ids below, then narrow to them. diff --git a/api/catalog/views/components.py b/api/catalog/views/components.py index bd0fb77..556b26b 100644 --- a/api/catalog/views/components.py +++ b/api/catalog/views/components.py @@ -69,6 +69,10 @@ def initial(self, request, *args, **kwargs): DRF reads `ordering` as an attribute, so a method cannot supply it. It does not check a default against `ordering_fields` either, so neither branch below needs an entry there. + + There is one sort control and one label, Smart, on every list. A + separate "best match" would name a ranking the search does not + work out. """ super().initial(request, *args, **kwargs) if request.query_params.get("q"): @@ -80,17 +84,6 @@ def initial(self, request, *args, **kwargs): # A service tab reads down the provider's own page. self.ordering = ["status_page_order"] - def filter_queryset(self, queryset): - """Narrow to what the caller typed, before the backends run. - - One control and one label, Smart, on every list. A separate - "best match" would name a ranking the search does not work out. - """ - q = self.request.query_params.get("q") - if q: - queryset = queryset.search(q) - return super().filter_queryset(queryset) - @extend_schema_view( get=extend_schema(responses={200: ComponentSerializer, 404: ErrorSerializer}) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index a145939..3c94e6a 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -1177,41 +1177,3 @@ def test_the_components_link_opens_exactly_what_it_counts(admin_client): link = related_item(admin_user, service, "Components")["link"] assert admin_client.get(link).context["cl"].result_count == 1 - - -@pytest.mark.django_db -def test_component_admin_fieldsets_order_featured_and_archive(): - # Featured moved to the base section. Archive fields moved to their - # own section. Verify the fieldsets layout against regressions. - from catalog.models import ServiceComponent - - model_admin = admin.site._registry[ServiceComponent] - fieldsets = model_admin.get_fieldsets(None) - - # Four sections: base, Position, Archive, audit. - assert len(fieldsets) == 4, f"Expected 4 fieldsets, got {len(fieldsets)}" - - # First section (base): has is_featured, not archive fields. - base_section = fieldsets[0] - assert base_section[0] is None, "First section should have no title" - base_fields = base_section[1]["fields"] - assert "is_featured" in base_fields, "is_featured must be in base section" - assert "is_archived" not in base_fields, "is_archived must not be in base section" - assert "archived_at" not in base_fields, "archived_at must not be in base section" - - # Second section: Position (unchanged). - position_section = fieldsets[1] - assert "Position" in str(position_section[0]), "Second section must be Position" - assert "parent" in position_section[1]["fields"], ( - "Position section must have parent" - ) - - # Third section: Archive with archive fields. - archive_section = fieldsets[2] - assert "Archive" in str(archive_section[0]), "Third section must be Archive" - archive_fields = archive_section[1]["fields"] - assert "is_archived" in archive_fields, "is_archived must be in Archive section" - assert "archived_at" in archive_fields, "archived_at must be in Archive section" - - # Fourth section: audit (checked by presence). - assert fieldsets[3] is not None, "Fourth section (audit) must exist" From 8582b114ef5223d270733aa852f81627c0b8cefb Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:16:00 -0700 Subject: [PATCH 71/94] feat: search matches the parent's name as well A person types the name of the group a part sits under. Searching `messaging` found Programmable Messaging and not the SMS row inside it, which is the row they were looking for. `parent__name` is a join read at query time, against the live row. The weighted vector this replaces stored ancestor names, so every rename rewrote the subtree below it. That is the cost the change does not carry back. One level only, because two would need the ancestry stored again. The docstring and the spec both say so. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/models.py | 16 +++++++++--- api/tests/test_component_api.py | 26 +++++++++++++++++++ docs/api/openapi.yaml | 7 ++--- .../2026-09-03-statusboard-client-design.md | 18 ++++++++----- 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/api/catalog/models.py b/api/catalog/models.py index d58ef3a..d84a919 100644 --- a/api/catalog/models.py +++ b/api/catalog/models.py @@ -232,9 +232,15 @@ def visible(self): def search(self, q): """Narrow to what every typed word describes. - A word matches the component's own name or its service's, so - `twilio` reaches Twilio's SMS row. Several words are an AND, so - `twilio sms` reaches that row and not its siblings. + A word matches the component's own name, its service's or its + parent's. So `twilio` reaches Twilio's SMS row, and `messaging` + reaches it through the group above it. Several words are an AND, + so `twilio sms` reaches that row and not its siblings. + + One level up, and no further. A component three deep is not + found by its grandparent's name. `parent__name` is a join read + at query time, so a rename needs nothing rewritten. Storing the + ancestry is what forced a subtree rewrite on every rename. The result is unordered. The list view decides what leads a search. An ordering backend runs after this and would discard @@ -243,7 +249,9 @@ def search(self, q): rows = self for term in q.split(): rows = rows.filter( - models.Q(name__icontains=term) | models.Q(service__name__icontains=term) + models.Q(name__icontains=term) + | models.Q(service__name__icontains=term) + | models.Q(parent__name__icontains=term) ) return rows diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py index b4d175d..f5c01ec 100644 --- a/api/tests/test_component_api.py +++ b/api/tests/test_component_api.py @@ -64,6 +64,32 @@ def test_q_reaches_a_component_through_its_services_name(client, tree): assert [r["name"] for r in response.json()["results"]] == ["SMS"] +def test_q_reaches_a_component_through_its_parents_name(client, tree): + # Searching "messaging" must find SMS as well as the group above + # it. A part is named for what it does, not for what it sits under. + response = client.get(reverse("component-list"), {"q": "messaging"}) + assert {r["name"] for r in response.json()["results"]} == { + "Programmable Messaging", + "SMS", + } + + +def test_q_combines_a_parents_name_with_the_components_own(client, tree): + # Every word must match, and any of the three names may satisfy it. + # "messaging sms" is how a person narrows to one part of a group. + response = client.get(reverse("component-list"), {"q": "messaging sms"}) + assert [r["name"] for r in response.json()["results"]] == ["SMS"] + + +def test_q_does_not_reach_past_one_level(client, tree): + # The honest limit. A grandparent's name is not stored on the row, + # and matching it would need the ancestry back. + service, _, _, leaf = tree + ComponentFactory(service=service, name="Verify", parent=leaf) + response = client.get(reverse("component-list"), {"q": "programmable verify"}) + assert response.json()["results"] == [] + + def test_service_narrows_to_one_services_parts(client, tree): # A service's Components tab. The rollup is excluded there, # because the header already carries the service's status. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index b28995c..9b3e72f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -309,9 +309,10 @@ paths: `service`, a component's descendants with `ancestor`, an event's affected list with `event`. - `q` matches part of a component's name or its service's name, so `twilio` finds `SMS`. - Several words are an AND. The service rollup leads the results. Without `q` the default - is `suggested`, except under `service`, where it is `status_page_order`. + `q` matches part of a component's name, its service's name, or its parent's. So `twilio` + finds `SMS`, and so does `messaging`. The parent is one level: a grandparent's name does + not reach it. Several words are an AND. The service rollup leads the results. Without `q` + the default is `suggested`, except under `service`, where it is `status_page_order`. Archived components are never listed. An event whose only affected component was archived returns an empty page. diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md index b60cb11..86927e7 100644 --- a/docs/specs/2026-09-03-statusboard-client-design.md +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -233,7 +233,7 @@ level, and `parent` would name a query this does not run. | --- | --- | | `GET /catalog/components/` with no `q` | `suggested` | | `GET /catalog/components/?service=` | `status_page_order` | -| `GET /catalog/components/` with `q` | `SearchRank` | +| `GET /catalog/components/` with `q` | `(-is_overall, *suggested)` | | `GET /events/` | `-starts_at` | Every one of them is labelled Smart in the interface. @@ -444,9 +444,15 @@ the same question. ## 8. Search -`q` matches part of a component's own name or its service's name. Searching `twilio` finds -`SMS`, because Twilio publishes it. Searching `twilio sms` finds it too: every word must -match, and either column may satisfy it. Matching ignores case. +`q` matches part of a component's own name, its service's name, or its parent's. Searching +`twilio` finds `SMS`, because Twilio publishes it. Searching `messaging` finds it too, +because Programmable Messaging is the group above it. Searching `twilio sms` finds it as +well: every word must match, and any of the three names may satisfy it. Matching ignores +case. + +The parent is one level, and no further. A component three deep is not found by its +grandparent's name. `parent__name` is a join read at query time, against the live row, so a +rename keeps itself in step. That is what separates it from the stored ancestry below. It is `icontains` per word, not a `tsvector`. Measured on 4,800 components across 60 services, larger than realistic, a stored weighted vector with a GIN index answers in about @@ -465,8 +471,8 @@ there is nothing to keep in step. `ServiceComponent` becomes a plain model with - **Relevance ranking.** There is no score, so nothing sorts by closeness of match. The rollup leads instead, by `-is_overall`, which is what stops a popular service flooding the list with its eighty parts. -- **Matching through an intermediate group.** A leaf is no longer found by the name of a - group above it, only by its own name and its service's. +- **Matching through the whole ancestry.** A leaf is found by its parent's name, but no + higher. A group two or more levels above it no longer reaches it. The service `search_fields` go with the service list. From 2113b5fe1e6841e9c7a2df893b3e223329c1980e Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:17:24 -0700 Subject: [PATCH 72/94] fix: the phase filter reads as the two choices it has `filter_phase` named `CLOSED` and fell through to a trailing exclude. That read as a fallback for the no-value case, so a reader asked whether an absent `phase` means open only. It does not: django-filter skips a method for an empty value, and everything comes back. Both branches now name the choice they serve, and the docstring says what an absent parameter does. Nothing asserted the no-parameter case, so a change that made open the default would have been silent. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/status/filters.py | 15 ++++++++++++--- api/tests/test_event_api.py | 18 ++++++++++++++++++ docs/api/openapi.yaml | 2 +- .../2026-09-03-statusboard-client-design.md | 3 ++- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/api/status/filters.py b/api/status/filters.py index ec5646f..b5ab0bc 100644 --- a/api/status/filters.py +++ b/api/status/filters.py @@ -27,9 +27,18 @@ class Meta: fields = {"kind": ["exact"]} def filter_phase(self, queryset, name, value): - if value == EventPhaseState.CLOSED: - return queryset.filter(phase__in=CLOSED_PHASES) - return queryset.exclude(phase__in=CLOSED_PHASES) + """Narrow to the open phases, or to the terminal ones. + + An absent `phase` returns both. django-filter skips a method + when the value is empty, so nothing here runs and no default + applies. + + `ChoiceFilter` admits `open` and `closed` only. The last return + is the closed case, not a fallback. + """ + if value == EventPhaseState.OPEN: + return queryset.exclude(phase__in=CLOSED_PHASES) + return queryset.filter(phase__in=CLOSED_PHASES) def filter_dashboard(self, queryset, name, value): """Everything posted across the services on one board. diff --git a/api/tests/test_event_api.py b/api/tests/test_event_api.py index 6885b00..df937f3 100644 --- a/api/tests/test_event_api.py +++ b/api/tests/test_event_api.py @@ -154,6 +154,24 @@ def test_phase_open_and_closed_draw_one_line(client): assert {r["title"] for r in closed} == {"Done", "Finished"} +def test_no_phase_parameter_returns_every_event(client): + # Omitting `phase` is not a filter. The code reads as though open + # were the default, and a change that made it one would be silent. + service = ServiceFactory() + component = ComponentFactory(service=service) + _event(service, component, external_id="1", title="Open") + _event( + service, + component, + external_id="2", + title="Done", + phase=IncidentPhase.RESOLVED, + ) + + results = client.get(reverse("event-list")).json()["results"] + assert {r["title"] for r in results} == {"Open", "Done"} + + def test_dashboard_narrows_to_what_you_track(authenticated_client, board): # Home's Updates tab. Everything posted across the services on # your board, and nothing else. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9b3e72f..710f98f 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -388,7 +388,7 @@ paths: - { name: service, in: query, schema: { type: string }, description: Service slug } - { name: component, in: query, schema: { type: string, format: uuid } } - { name: kind, in: query, schema: { type: string, enum: [incident, maintenance] } } - - { name: phase, in: query, schema: { type: string, enum: [open, closed] }, description: 'Render the label from `/meta/` `enums.event_phase_state`.' } + - { name: phase, in: query, schema: { type: string, enum: [open, closed] }, description: 'Omit it and both come back. Render the label from `/meta/` `enums.event_phase_state`.' } - { name: ordering, in: query, schema: { type: string, enum: [-starts_at, starts_at, ends_at, -ends_at], default: -starts_at } } - { $ref: '#/components/parameters/Fields' } - { $ref: '#/components/parameters/Cursor' } diff --git a/docs/specs/2026-09-03-statusboard-client-design.md b/docs/specs/2026-09-03-statusboard-client-design.md index 86927e7..53ed98c 100644 --- a/docs/specs/2026-09-03-statusboard-client-design.md +++ b/docs/specs/2026-09-03-statusboard-client-design.md @@ -198,7 +198,8 @@ and `/meta/` publishes the labels. ### Phase is a declared filter `phase` accepts `open` and `closed`. It draws `CLOSED_PHASES` from `status/choices.py`, so -the client never restates which phases are terminal. +the client never restates which phases are terminal. Omitting it returns both, so the feed +is not an open-only list that a client has to opt out of. ### A tab badge is the collection's total From 59b4aa93171ac38fcefebc2aa8b570a505aed94c Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:18:22 -0700 Subject: [PATCH 73/94] docs: say that the search sort is a default, not an override `initial` sets `ordering` when `q` or `service` is present, and the comments explained the mechanism without naming these as defaults. A reader took it twice as forcing a caller's sort. It does not: DRF reads `?ordering=` afterwards and it wins. A test now holds that. Losing it would leave the sort control on the screen doing nothing, and nothing would say so. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/views/components.py | 3 +++ api/tests/test_component_api.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/api/catalog/views/components.py b/api/catalog/views/components.py index 556b26b..d016a75 100644 --- a/api/catalog/views/components.py +++ b/api/catalog/views/components.py @@ -66,6 +66,9 @@ class ComponentListView(ComponentQueryMixin, generics.ListAPIView): def initial(self, request, *args, **kwargs): """Pick the default sort from what the caller asked for. + Both branches set a default, and neither overrides anybody. + `?ordering=` is read after this and wins when it is present. + DRF reads `ordering` as an attribute, so a method cannot supply it. It does not check a default against `ordering_fields` either, so neither branch below needs an entry there. diff --git a/api/tests/test_component_api.py b/api/tests/test_component_api.py index f5c01ec..683ced8 100644 --- a/api/tests/test_component_api.py +++ b/api/tests/test_component_api.py @@ -64,6 +64,20 @@ def test_q_reaches_a_component_through_its_services_name(client, tree): assert [r["name"] for r in response.json()["results"]] == ["SMS"] +def test_an_explicit_ordering_beats_the_default_a_search_picks(client, tree): + # The view sets a default sort when `q` is present. It is a + # default: a caller who names one reads their own order. Losing + # that leaves the sort control on the screen doing nothing. + response = client.get( + reverse("component-list"), {"q": "twilio", "ordering": "name"} + ) + assert [r["name"] for r in response.json()["results"]] == [ + "Programmable Messaging", + "SMS", + "Twilio", + ] + + def test_q_reaches_a_component_through_its_parents_name(client, tree): # Searching "messaging" must find SMS as well as the group above # it. A part is named for what it does, not for what it sits under. From 2a4bfb29e94e522a568a208bd9913df015a644bf Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:21:36 -0700 Subject: [PATCH 74/94] chore: standardize the just recipes so dev starts what it needs just dev ran the server and poller but never started Postgres or Redis and never migrated, so a fresh worktree failed unexplained. It now depends on up and migrate. check-pages is renamed to probe-pages, since sharing a prefix with the merge gate (check) was misleading. down and teardown, which overlapped, merge into one clean recipe. test-cov is gone: check already owns the coverage gate, and test now takes *args, so the only reason test-cov existed (test couldn't be filtered) no longer holds. test-bin folds into test, and env, sync, seed and image (steps of init and check, never typed on their own) are marked [private]: still runnable, hidden from --list. Recipes are grouped (Daily, Lifecycle, Data, Release) so `just --list` reads as a short, obvious set instead of an alphabetical wall of 23 names. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 38 +++++++++++- README.md | 5 +- justfile | 183 +++++++++++++++++++++++++++++------------------------- 3 files changed, 137 insertions(+), 89 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4f0e123..beab02d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,11 +116,45 @@ Every behaviour change carries a test. The suite runs with no network: adapters are tested against recorded fixtures. A test's comment says what would break if the assertion failed. +If the answer is "nothing", the test does not earn its place. + +**Assert on behaviour, not arrangement.** Field order, section titles and +`list_display` contents change for cosmetic reasons. A test that pins +them fails on a harmless edit and catches no defect. One pinned the four +component fieldsets in order. Moving two lines then cost a test edit, and +it had never caught a bug. + +**A test fails only when something is broken.** A reasonable refactor +breaks a wrong test and nothing else. + +**Never restate the implementation.** A test that computes its expected +value the way the code does passes while both are wrong. Write the +expected value out. + +**No test-only production code.** No argument, method, flag or branch +exists so that a test can reach something. A path no caller reaches is a +path the test should not reach either. + +**Enter through the front door.** Use the API client, the admin client or +the management command. Call an internal directly only when no pathway +reaches it. + +**Every assertion can fail.** Watch it fail once, before the code exists. +`assert fieldsets[3] is not None` held on every possible input. + +**Test what this project decided.** Django and DRF are already tested. +That a `CharField` stores a string is theirs. That a poller claims an +incident inside the window is ours. + +**In the admin, test the logic, not the form.** Worth pinning: a +read-only field the model maintains, a queryset the page filters, what +`list_editable` writes, which user `created_by` records. Not which +fields appear, or in what order. ```sh -just test # the suite -just test-cov # with the coverage gate +just test # the suite, host and bin/ scripts together just lint # ruff check and format +just check # every check CI runs, coverage gate included ``` ## Database diff --git a/README.md b/README.md index 9ca506b..414d09d 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,14 @@ Each worktree gets its own ports and database. | | | | --- | --- | | `just init` | services, dependencies, migrations, admin, hooks | -| `just dev` | server and poller together | +| `just dev` | postgres, redis, migrations, then the server and poller | | `just test` | the suite, on the host | | `just check` | everything CI runs, in the image CI runs it in | | `just lint` | fix and format | | `just info` | this worktree's ports and database | | `just reset` | drop the database and build it again | -| `just check-pages` | probe every recorded status page | +| `just clean` | stop this checkout's containers | +| `just probe-pages` | probe every recorded status page | | `just release v0.1.0` | tag, build, sign, deploy | | `just deploy` | redeploy the current image | diff --git a/justfile b/justfile index 43b2c54..be224c1 100644 --- a/justfile +++ b/justfile @@ -8,18 +8,56 @@ wt := 'eval "$(python3 bin/worktree-env.py)"' # List the recipes. Keep first: bare `just` runs whatever recipe comes first. default: - @just --list + @just --list --unsorted -# One-time setup. Every step it calls is runnable on its own. -init: env up sync migrate seed - pre-commit install - @{{wt}} ; cd api && uv run python manage.py collectstatic --noinput +# Applying migrations here is a little surprising: starting a server +# that also changes the database. The human asked for exactly this, +# since a fresh worktree that only warns still fails unexplained. -# Write .env.local, asking for the admin once. Worktrees copy main's. -env: - @python3 bin/make-env.py +# Start Postgres and Redis, migrate, then run the server and the poller. Ctrl-C stops both. +[group('Daily')] +dev: up migrate + @{{wt}} ; trap 'kill 0' EXIT INT TERM ; \ + echo "http://$WORKTREE_SLUG.localhost:$DJANGO_PORT/" ; \ + cd api ; \ + uv run celery -A api worker -B -l info & \ + uv run python manage.py runserver 0.0.0.0:$DJANGO_PORT + +# Run the test suite, host and bin/ scripts together. Arguments pass to both, so `just test -k foo` filters each. +[group('Daily')] +test *args: + @{{wt}} ; cd api && uv run pytest -n auto {{args}} + @python3 -m unittest discover -s bin/tests -v {{args}} + +# Fix lint and format the code. +[group('Daily')] +lint: + cd api && uv run ruff check --fix . && uv run ruff format . + +# Slower than `just test`, and the answer that counts: it proves the +# image rather than the host. Run it before opening a pull request. +# The two stdlib-only checks run on the host, because a container costs +# more than it saves. `bin/tests/test_merge_gate.py` holds the two lists +# together, in one direction: everything here also runs in CI. +# +# CI runs one more. It scans the image with Trivy for HIGH and CRITICAL +# CVEs and fails the merge on a hit. That needs the network and a +# vulnerability database, so it stays out of this recipe. + +# Every CI check but the image scan, each where CI runs it. +[group('Daily')] +check: image + docker compose -f docker-compose.test.yml run --rm pytest -q -n auto --cov-fail-under=85 + docker compose -f docker-compose.test.yml run --rm ruff + docker compose -f docker-compose.test.yml run --rm docs + @python3 bin/check_prose.py + @python3 -m unittest discover -s bin/tests -v + docker compose -f docker-compose.test.yml run --rm --entrypoint sh pytest \ + -c 'python manage.py makemigrations --check --dry-run' + docker compose -f docker-compose.test.yml down -v # Show this worktree's ports, database and compose project. +[group('Daily')] info: @{{wt}} ; echo "worktree $WORKTREE_SLUG" ; \ echo "compose $COMPOSE_PROJECT_NAME" ; \ @@ -28,21 +66,40 @@ info: echo "server http://$WORKTREE_SLUG.localhost:$DJANGO_PORT/" ; \ echo "client http://$WORKTREE_SLUG.localhost:$CLIENT_PORT (reserved)" +# Check comments and docstrings against AGENTS.md. +[group('Daily')] +prose *args: + @python3 bin/check_prose.py {{args}} + +# One-time setup. Every step it calls is runnable on its own. +[group('Lifecycle')] +init: env up sync migrate seed + pre-commit install + @{{wt}} ; cd api && uv run python manage.py collectstatic --noinput + # Start Postgres and Redis for this worktree, waiting until they are healthy. +[group('Lifecycle')] up: @{{wt}} ; docker compose up -d --wait ; \ echo "postgres localhost:$POSTGRES_HOST_PORT redis localhost:$REDIS_HOST_PORT" -# Stop them, keeping the data. `just down -v` drops this worktree's database. -down *args: - @{{wt}} ; docker compose down {{args}} +# Apply migrations to this worktree's database. +[group('Lifecycle')] +migrate: + @{{wt}} ; cd api && uv run python manage.py migrate + +# Drop this worktree's database and build it again from nothing. +[group('Lifecycle')] +reset: + @{{wt}} ; docker compose down -v ; just up ; just migrate ; just seed-live # Also clears what an aborted `just check` left, under the compose # project that file uses. Its database is a tmpfs, so it holds nothing. # Add `-v` to drop this checkout's own database too. # Stop this checkout's containers and remove its networks. -teardown *args: +[group('Lifecycle')] +clean *args: @{{wt}} ; DIR=$(basename "$PWD") ; \ docker compose down --remove-orphans {{args}} ; \ docker compose -p "$DIR" -f docker-compose.test.yml down -v --remove-orphans ; \ @@ -53,81 +110,21 @@ teardown *args: echo "[just] git branch -d $(git rev-parse --abbrev-ref HEAD)" ; \ fi -# Install Python dependencies. -sync: - cd api && uv sync - -# Apply migrations to this worktree's database. -migrate: - @{{wt}} ; cd api && uv run python manage.py migrate - -# Create the admin in this worktree's database, from .env.local. Never asks. -seed: - @{{wt}} ; cd api && uv run python manage.py seed_admin - -# Fetches three real status pages, so it needs the network. - -# Fill an empty local database: admin, a small catalog, one tracked service. -seed-dev: +# Fetches real status pages, so it needs the network. Fills an empty database: admin, a small catalog, one tracked service. +[group('Data')] +seed-live: @{{wt}} ; cd api && uv run python manage.py seed_dev -# Drop this worktree's database and build it again from nothing. -reset: - @{{wt}} ; docker compose down -v ; just up ; just migrate ; just seed-dev - # Probe the recorded status pages and report any that moved or broke. -check-pages *args: +[group('Data')] +probe-pages *args: @{{wt}} ; cd api && uv run python manage.py check_status_pages {{args}} -# Run the test suite. -test: - @{{wt}} ; cd api && uv run pytest -n auto - -# Run the test suite with the coverage gate. -test-cov: - @{{wt}} ; cd api && uv run pytest -n auto --cov-fail-under=85 - -# Fix lint and format the code. -lint: - cd api && uv run ruff check --fix . && uv run ruff format . - -# Check comments and docstrings against AGENTS.md. -prose *args: - @python3 bin/check_prose.py {{args}} - -# Test bin/ scripts. Stdlib only, so it needs no venv and no Django. -test-bin: - @python3 -m unittest discover -s bin/tests -v - -# Slower than `just test`, and the answer that counts: it proves the -# image rather than the host. Run it before opening a pull request. -# The two stdlib-only checks run on the host, because a container costs -# more than it saves. `bin/tests/test_merge_gate.py` holds the two lists -# together, in one direction: everything here also runs in CI. -# -# CI runs one more. It scans the image with Trivy for HIGH and CRITICAL -# CVEs and fails the merge on a hit. That needs the network and a -# vulnerability database, so it stays out of this recipe. - -# Every CI check but the image scan, each where CI runs it. -check: image - docker compose -f docker-compose.test.yml run --rm pytest -q -n auto --cov-fail-under=85 - docker compose -f docker-compose.test.yml run --rm ruff - docker compose -f docker-compose.test.yml run --rm docs - @python3 bin/check_prose.py - @just test-bin - docker compose -f docker-compose.test.yml run --rm --entrypoint sh pytest \ - -c 'python manage.py makemigrations --check --dry-run' - docker compose -f docker-compose.test.yml down -v - -# Build the deployment image. `check` runs against this. -image: - docker build -t statusboard-api:test api - # The tag triggers release.yml: build, sign, smoke test, deploy. A tag # left behind by a failed push is reused rather than duplicated. # Tag and push a release (v1.2.3, v1.2.3-rc.1). Ships to production. +[group('Release')] release VERSION: @set -e ; \ VERSION="{{VERSION}}" ; \ @@ -177,6 +174,7 @@ release VERSION: # only ever deploy its own stack. # Redeploy the current image without cutting a release. +[group('Release')] deploy: @set -e ; \ APP="${FORGEJO_DEPLOY_APP:-}" ; \ @@ -200,10 +198,25 @@ deploy: fi ; \ echo "[deploy] queued. Watch: $HOST/${FORGEJO_REPO}/actions" -# Run the app: server and poller together. Ctrl-C stops both. -dev: - @{{wt}} ; trap 'kill 0' EXIT INT TERM ; \ - echo "http://$WORKTREE_SLUG.localhost:$DJANGO_PORT/" ; \ - cd api ; \ - uv run celery -A api worker -B -l info & \ - uv run python manage.py runserver 0.0.0.0:$DJANGO_PORT +# Everything below is a step of `init` or `check`, not a command a +# person reaches for. `just ` still runs each one directly. + +# Write .env.local, asking for the admin once. Worktrees copy main's. +[private] +env: + @python3 bin/make-env.py + +# Install Python dependencies. +[private] +sync: + cd api && uv sync + +# Create the admin in this worktree's database, from .env.local. Never asks. +[private] +seed: + @{{wt}} ; cd api && uv run python manage.py seed_admin + +# Build the deployment image. `check` runs against this. +[private] +image: + docker build -t statusboard-api:test api From f183522a783be20dd58dfa36577852ef8961cd64 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:22:41 -0700 Subject: [PATCH 75/94] docs: what a test has to earn to stay in the suite The suite had one rule: every behaviour change carries a test. Nothing said what a test should assert, so tests grew that pin arrangement. One pinned four admin fieldsets in an exact order and ended on an assertion that no input could fail. Moving two lines then cost a test edit, and it had never caught a bug. Eight principles now say what earns a place: behaviour over arrangement, no restated implementation, no test-only production code, and every assertion able to fail. Also repoints two comments at recipes that still exist, after the just recipes were renamed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/common/admin.py | 2 +- api/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/common/admin.py b/api/common/admin.py index 5eb78bd..fda6cd1 100644 --- a/api/common/admin.py +++ b/api/common/admin.py @@ -343,7 +343,7 @@ class PollerWrittenAdmin(BaseModelAdmin): Only the three permissions are refused. Django still renders the detail page read-only, so a poll error stays readable. - Seed a database with `just seed-dev`, which polls for real. + Seed a database with `just seed-live`, which polls for real. """ def has_add_permission(self, request, obj=None): diff --git a/api/pyproject.toml b/api/pyproject.toml index 369c1bd..e38992b 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -44,4 +44,4 @@ ignore = ["RUF012"] DJANGO_SETTINGS_MODULE = "api.settings" python_files = ["test_*.py"] addopts = "--cov=. --cov-report=term-missing" -# The gate is in `just test-cov`. Here it fails every run until the last task. +# The gate is in `just check`. Here it would fail on every partial run. From 3d1e8d0fff752b3f27c1f2883d569fe8242417cd Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:33:51 -0700 Subject: [PATCH 76/94] refactor: apply_fetch requires the run that produced the data Both production callers already passed one. The default reached only tests, which then wrote status and event rows with no run. That is a shape production never creates, so the run-stamping went untested. It also left the cross-service guard half dead. Its skip path ran only under tests, so a real run for the wrong service was checked by nothing the scheduler runs. The tests now build a real run through `tests.factories.poll_run`. One test moved off the removed default: a reading still outlives the run that stamped it, because the column is SET_NULL. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/polling/reconcile.py | 11 ++--- api/tests/factories.py | 15 +++++++ api/tests/test_cross_service.py | 2 + api/tests/test_reconcile.py | 78 +++++++++++++++++++++------------ 4 files changed, 72 insertions(+), 34 deletions(-) diff --git a/api/polling/reconcile.py b/api/polling/reconcile.py index c1c405d..30e4dae 100644 --- a/api/polling/reconcile.py +++ b/api/polling/reconcile.py @@ -8,7 +8,7 @@ @transaction.atomic -def apply_fetch(service, components, events, source, run=None): +def apply_fetch(service, components, events, source, run): """Write one adapter fetch to the database. A poll is a reconciliation, not a status read. @@ -16,7 +16,8 @@ def apply_fetch(service, components, events, source, run=None): An unchanged severity leaves the open status row alone. `run` is the PollRun that produced this data. Stamping it makes a - reading traceable back to the fetch that wrote it. + reading traceable back to the fetch that wrote it. Every fetch has + one, so nothing here writes a reading with no provenance. Nobody types any of this, so the system account signs every row. A blank author reads the same as one that was lost, and a component @@ -25,7 +26,7 @@ def apply_fetch(service, components, events, source, run=None): # A poll writes to the service it read. Nothing in the arguments # ties them together. A run from another poller would file one # service's readings under another. - if run is not None and run.poller.service_id != service.pk: + if run.poller.service_id != service.pk: raise ValueError( f"{run} polled {run.poller.service}, not {service}. " "A poll writes to the service it read." @@ -93,7 +94,7 @@ def _archive_vanished(service, components, author): ).update(is_archived=True, archived_at=timezone.now(), updated_by=author) -def _write_statuses(components, rows, source, author, run=None): +def _write_statuses(components, rows, source, author, run): now = timezone.now() for incoming in components: row = rows[incoming.external_id] @@ -139,7 +140,7 @@ def _affected(service, rows, external_ids): return named -def _upsert_events(service, events, rows, author, run=None): +def _upsert_events(service, events, rows, author, run): for incoming in events: event, _ = ServiceEvent.objects.update_or_create( service=service, diff --git a/api/tests/factories.py b/api/tests/factories.py index 27f75b7..c71099c 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -73,6 +73,21 @@ class Meta: name = factory.Sequence(lambda n: f"Component {n}") +def poll_run(service, **kwargs): + """The fetch that a test says wrote its rows. + + `apply_fetch` takes one, and refuses a run from another service's + poller. The run has to belong to the service being reconciled. + + The status page supplies the url and the provider. A test about the + reconciliation does not have to make one. + """ + from polling.models import PollRun + + page = getattr(service, "status_page", None) or StatusPageFactory(service=service) + return PollRun.open(service.poller, page.url, page.provider, **kwargs) + + def track(component, user=None): """Track a component, the way somebody using the app would. diff --git a/api/tests/test_cross_service.py b/api/tests/test_cross_service.py index 08832e8..9d652e9 100644 --- a/api/tests/test_cross_service.py +++ b/api/tests/test_cross_service.py @@ -179,6 +179,7 @@ def test_an_event_keeps_a_component_the_provider_stopped_listing(two_services): ) ], StatusSource.PROVIDER, + _run(mine), ) event = ServiceEvent.objects.get(external_id="inc-1") @@ -205,6 +206,7 @@ def test_a_poll_invents_no_component(two_services): ) ], StatusSource.PROVIDER, + _run(mine), ) assert not ServiceComponent.objects.filter(external_id="never-seen").exists() diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index 1cfd146..22aec5e 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -16,7 +16,7 @@ StatusSource, ) from status.models import ComponentStatus, ServiceEvent -from tests.factories import ComponentFactory, ServiceFactory, ancestry +from tests.factories import ComponentFactory, ServiceFactory, ancestry, poll_run def _component(external_id="a", name="SMS", severity=Severity.OPERATIONAL, **kw): @@ -33,7 +33,7 @@ def _component(external_id="a", name="SMS", severity=Severity.OPERATIONAL, **kw) @pytest.mark.django_db def test_a_new_component_is_created(): service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) assert ( ServiceComponent.objects.filter(service=service, external_id="a").count() == 1 ) @@ -44,7 +44,11 @@ def test_a_rename_updates_the_existing_row_rather_than_creating_one(): service = ServiceFactory() ComponentFactory(service=service, external_id="a", name="SMS") apply_fetch( - service, [_component(name="Programmable Messaging")], [], StatusSource.PROVIDER + service, + [_component(name="Programmable Messaging")], + [], + StatusSource.PROVIDER, + poll_run(service), ) rows = ServiceComponent.objects.filter(service=service) assert rows.count() == 1 @@ -56,7 +60,7 @@ def test_a_vanished_component_is_archived_not_deleted(): # Someone may be tracking it. Deleting would silently remove it from their board. service = ServiceFactory() ComponentFactory(service=service, external_id="gone", name="Old") - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) old = ServiceComponent.objects.get(service=service, external_id="gone") assert old.archived_at is not None assert ServiceComponent.objects.filter(service=service).count() == 2 @@ -66,7 +70,7 @@ def test_a_vanished_component_is_archived_not_deleted(): def test_a_component_that_reappears_is_unarchived(): service = ServiceFactory() ComponentFactory(service=service, external_id="a", archived_at=timezone.now()) - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) assert ( ServiceComponent.objects.get(service=service, external_id="a").archived_at is None @@ -81,6 +85,7 @@ def test_the_parent_child_structure_is_refreshed(): [_component("parent", "Group"), _component("child", "SMS", parent="parent")], [], StatusSource.PROVIDER, + poll_run(service), ) child = ServiceComponent.objects.get(service=service, external_id="child") assert child.parent.external_id == "parent" @@ -89,8 +94,8 @@ def test_the_parent_child_structure_is_refreshed(): @pytest.mark.django_db def test_an_unchanged_severity_does_not_open_a_second_status_row(): service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) component = ServiceComponent.objects.get(service=service, external_id="a") assert ComponentStatus.objects.filter(component=component).count() == 1 @@ -98,9 +103,13 @@ def test_an_unchanged_severity_does_not_open_a_second_status_row(): @pytest.mark.django_db def test_a_changed_severity_closes_the_old_row_and_opens_a_new_one(): service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) apply_fetch( - service, [_component(severity=Severity.MAJOR_OUTAGE)], [], StatusSource.PROVIDER + service, + [_component(severity=Severity.MAJOR_OUTAGE)], + [], + StatusSource.PROVIDER, + poll_run(service), ) component = ServiceComponent.objects.get(service=service, external_id="a") rows = ComponentStatus.objects.filter(component=component).order_by("started_at") @@ -120,7 +129,7 @@ def test_an_event_is_upserted_on_its_provider_id(): phase=IncidentPhase.INVESTIGATING, starts_at=timezone.now(), ) - apply_fetch(service, [], [event], StatusSource.PROVIDER) + apply_fetch(service, [], [event], StatusSource.PROVIDER, poll_run(service)) apply_fetch( service, [], @@ -134,6 +143,7 @@ def test_an_event_is_upserted_on_its_provider_id(): ) ], StatusSource.PROVIDER, + poll_run(service), ) rows = ServiceEvent.objects.filter(service=service) assert rows.count() == 1 @@ -158,6 +168,7 @@ def test_an_event_is_linked_to_the_components_it_names(): ) ], StatusSource.PROVIDER, + poll_run(service), ) event = ServiceEvent.objects.get(service=service) assert [c.external_id for c in event.affected_components.all()] == ["a"] @@ -173,6 +184,7 @@ def test_a_later_provider_post_joins_the_event_the_poll_opened(): [_component("a", severity=Severity.DEGRADED)], [], StatusSource.PROVIDER, + poll_run(service), ) ours = ServiceEvent.objects.get(service=service) assert ours.detected_by == EventSource.SYSTEM @@ -198,6 +210,7 @@ def test_a_later_provider_post_joins_the_event_the_poll_opened(): ) ], StatusSource.PROVIDER, + poll_run(service), ) event = ServiceEvent.objects.get(service=service) @@ -212,17 +225,8 @@ def test_a_later_provider_post_joins_the_event_the_poll_opened(): def test_a_run_stamps_what_it_wrote(): # A wrong or stale reading is otherwise untraceable: you can see what # it says and not where it came from. - from polling.models import PollRun - from tests.factories import PollerFactory, StatusPageFactory - service = ServiceFactory() - page = StatusPageFactory(service=service) - run = PollRun.objects.create( - poller=PollerFactory(service=service), - url=page.url, - provider=page.provider, - started_at=timezone.now(), - ) + run = poll_run(service) event = NormalisedEvent( external_id="inc-1", kind=EventKind.INCIDENT, @@ -237,11 +241,19 @@ def test_a_run_stamps_what_it_wrote(): @pytest.mark.django_db -def test_a_hand_seeded_row_has_no_run(): - # apply_fetch is called without one when nothing polled. +def test_a_polled_reading_survives_its_run_being_deleted(): + # The log is pruned on a retention window, and a reading is not. On + # CASCADE, forgetting last month's runs would take the history of + # every component with them. service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) - assert ComponentStatus.objects.get(component__service=service).poll_run is None + run = poll_run(service) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, run) + + run.delete() + + status = ComponentStatus.objects.get(component__service=service) + assert status.severity == Severity.OPERATIONAL + assert status.poll_run is None @pytest.mark.django_db @@ -271,7 +283,9 @@ def test_a_poll_signs_everything_it_writes(): ) ], ) - apply_fetch(service, [_component()], [event], StatusSource.PROVIDER) + apply_fetch( + service, [_component()], [event], StatusSource.PROVIDER, poll_run(service) + ) system = get_user_model().objects.system() for model in (ServiceComponent, ComponentStatus, ServiceEvent, EventUpdate): @@ -286,10 +300,16 @@ def test_a_second_poll_does_not_rewrite_who_made_the_row(): # A row records who made it once. Re-signing it on every poll would # make `created_at` and `created_by` disagree. service = ServiceFactory() - apply_fetch(service, [_component()], [], StatusSource.PROVIDER) + apply_fetch(service, [_component()], [], StatusSource.PROVIDER, poll_run(service)) first = ServiceComponent.objects.get() - apply_fetch(service, [_component(name="Renamed")], [], StatusSource.PROVIDER) + apply_fetch( + service, + [_component(name="Renamed")], + [], + StatusSource.PROVIDER, + poll_run(service), + ) again = ServiceComponent.objects.get() assert again.created_by_id == first.created_by_id @@ -307,14 +327,14 @@ def test_a_reparent_moves_the_whole_subtree(): _component(external_id="b", parent="a"), _component(external_id="c", parent="b"), ] - apply_fetch(service, nested, [], StatusSource.PROVIDER) + apply_fetch(service, nested, [], StatusSource.PROVIDER, poll_run(service)) moved = [ _component(external_id="a"), _component(external_id="b"), _component(external_id="c", parent="b"), ] - apply_fetch(service, moved, [], StatusSource.PROVIDER) + apply_fetch(service, moved, [], StatusSource.PROVIDER, poll_run(service)) assert ancestry(ServiceComponent.objects.get(external_id="c")) == [ ServiceComponent.objects.get(external_id="b").id From aa4a69aaa2944c5b387de41f89de25b5b5fa53b5 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:34:00 -0700 Subject: [PATCH 77/94] test: the poll schedule is checked against a written-out window The jitter bound was built from the constant the code uses. A wrong JITTER moved both, and the test still passed. It is now a tenth either side, written out. The one-poller test made the poller itself, through a factory that gets or creates. The count then held with the signal that makes it deleted. It also promised two boards with one watcher. It now tracks the service from two boards, and asks only that `due()` returns the poller once. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_polling.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/api/tests/test_polling.py b/api/tests/test_polling.py index 3613305..aa2c286 100644 --- a/api/tests/test_polling.py +++ b/api/tests/test_polling.py @@ -43,11 +43,11 @@ def test_a_poller_inside_its_cooldown_is_not_due(): @pytest.mark.django_db def test_two_boards_tracking_one_service_produce_one_poller(): - # The cooldown is on the service, not on a user or a board. - service = ServiceFactory(tracked=1) + # The cooldown is on the service, not on a user or a board. `due()` + # asks whether anybody tracks the service. Asked as a join instead, + # it would return this poller once per board and poll twice as often. + service = ServiceFactory(tracked=2) StatusPageFactory(service=service) - PollerFactory(service=service) - assert Poller.objects.filter(service=service).count() == 1 assert len(list(Poller.objects.due())) == 1 @@ -337,8 +337,9 @@ def test_an_import_schedules_the_same_way_a_poll_does(): # Two callers wrote this. The import set a bare interval, with no # backoff and no jitter. A hundred imported at once came due on the # same second. - from polling.models import JITTER - + # + # The window is a tenth either side, written out. Read from the + # constant, this would pass with the constant itself set wrong. poller = PollerFactory(service=ServiceFactory(tracked=1)) at = timezone.now() @@ -348,7 +349,7 @@ def test_an_import_schedules_the_same_way_a_poll_does(): assert poller.consecutive_failure_count == 0 interval = poller.effective_interval_seconds gap = (poller.next_at - at).total_seconds() - assert interval * (1 - JITTER) <= gap <= interval * (1 + JITTER) + assert interval * 0.9 <= gap <= interval * 1.1 @pytest.mark.django_db From 1396a0a661afec0904064f49da1dedcf18f82a79 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:34:06 -0700 Subject: [PATCH 78/94] test: the subject test asks about the header, not the copy It pinned the exact subject line beside the newline check. A copy edit then failed a test about header injection. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_auth.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/tests/test_auth.py b/api/tests/test_auth.py index 9c0f27a..2abdf51 100644 --- a/api/tests/test_auth.py +++ b/api/tests/test_auth.py @@ -154,9 +154,7 @@ def test_the_subject_is_one_line(): # A header cannot hold a newline, and a template file ends with one. APIClient().post(reverse("magic-link"), {"email": "a@b.com"}, format="json") - subject = mail.outbox[0].subject - assert "\n" not in subject - assert subject == "Your sign-in link for Statusboard" + assert "\n" not in mail.outbox[0].subject def test_no_app_means_no_link_rather_than_a_link_to_nowhere(settings): From 7a851a6d96678f64b0d284da91f0ad7854d84022 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:35:43 -0700 Subject: [PATCH 79/94] test: one test for a reading outliving its run, not two The rewrite of the hand-seeded test landed beside a test in test_polling that already proved the same SET_NULL behaviour. Two tests answering one question drift apart. The kept one prunes through `forget_old_poll_runs`, which is what deletes a run in production. The removed one called `delete()` directly, which nothing does. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_reconcile.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/api/tests/test_reconcile.py b/api/tests/test_reconcile.py index 22aec5e..789e12e 100644 --- a/api/tests/test_reconcile.py +++ b/api/tests/test_reconcile.py @@ -240,22 +240,6 @@ def test_a_run_stamps_what_it_wrote(): assert ServiceEvent.objects.get(service=service).poll_run == run -@pytest.mark.django_db -def test_a_polled_reading_survives_its_run_being_deleted(): - # The log is pruned on a retention window, and a reading is not. On - # CASCADE, forgetting last month's runs would take the history of - # every component with them. - service = ServiceFactory() - run = poll_run(service) - apply_fetch(service, [_component()], [], StatusSource.PROVIDER, run) - - run.delete() - - status = ComponentStatus.objects.get(component__service=service) - assert status.severity == Severity.OPERATIONAL - assert status.poll_run is None - - @pytest.mark.django_db def test_a_poll_signs_everything_it_writes(): """Nobody types any of this, so nothing is left without an author. From 90b785ed4bf14131f6d28eb566c7dc1bb1ec0ae2 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:43:42 -0700 Subject: [PATCH 80/94] test: assertions pin behaviour, not the markup around it A test that pins a section title, a card label or an Unfold DOM id fails on a cosmetic edit and catches no defect. Ten such tests went. Some restated a framework guarantee: Django's own `admin.E116` already proves every `list_filter` path resolves, and the stdlib owns the sort order of a version 7 key. The rest kept the assertion that can fail and dropped the one that cannot. A reading now asserts the poll run's own link, not a field label. The table search proves a service's name reaches the rows under it, which is what its comment always claimed. This stops a rename or an Unfold upgrade from costing a test edit. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_admin.py | 243 ++++++++++++---------------- api/tests/test_catalog_models.py | 1 - api/tests/test_common_models.py | 18 +-- api/tests/test_dashboards_models.py | 6 - api/tests/test_status_models.py | 15 -- 5 files changed, 105 insertions(+), 178 deletions(-) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 3c94e6a..8c1046d 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -108,16 +108,10 @@ def service_form_data(staff_client, **fields): def test_the_dashboard_leads_with_poller_health(staff_client): # The landing page answers "is polling healthy?". A stalled poller # shows every board a stale green, which is worse than showing nothing. - body = staff_client.get(reverse("admin:index")).content.decode() - for card in ( - "Services tracked", - "Behind schedule", - "Poll success", - "Stalest service", - ): - assert card in body, f"{card} missing from the dashboard" - for panel in ("Polls, last 24 hours", "Services tracked, last 30 days"): - assert panel in body, f"{panel} missing from the dashboard" + response = staff_client.get(reverse("admin:index")) + # `dashboard_callback` builds the cards. Unwired, the page is bare. + assert response.context["cards"] + body = response.content.decode() # Neither the catalog of every model nor a log of admin edits says # anything about whether polling works. assert "app-list" not in body @@ -131,13 +125,6 @@ def test_the_environment_is_named_on_every_page(staff_client): assert "Development" in staff_client.get(reverse("admin:index")).content.decode() -@pytest.mark.django_db -def test_the_brand_marks_are_wired(staff_client): - body = staff_client.get(reverse("admin:index")).content.decode() - assert "statusboard/logo-light.svg" in body - assert "statusboard/favicon.svg" in body - - @pytest.mark.django_db @pytest.mark.parametrize( "model", @@ -252,14 +239,6 @@ def test_derived_fields_are_not_on_the_form(staff_client, field): assert f'name="{field}"' not in body -@pytest.mark.django_db -def test_the_changelist_offers_an_import_button(staff_client): - body = staff_client.get( - reverse("admin:catalog_service_changelist") - ).content.decode() - assert "Import from URL" in body - - @pytest.mark.django_db def test_importing_from_the_admin_creates_the_service(staff_client, monkeypatch): from catalog.models import Service @@ -321,6 +300,9 @@ def test_events_can_be_filtered_by_phase(staff_client): An incident and a maintenance window move through different phases, which is why a plain ChoicesDropdownFilter cannot do this. + + The two option labels are what prove the filter is declared. `phase` + is a local field, so the queryset narrows with or without it. """ from django.utils import timezone @@ -376,7 +358,6 @@ def test_every_table_links_somewhere(staff_client, url_name): from django.utils import timezone - from catalog.models import Service from polling.models import PollRun from status.choices import EventKind, IncidentPhase, Severity, StatusSource from status.models import ComponentStatus, ServiceEvent @@ -407,20 +388,19 @@ def test_every_table_links_somewhere(staff_client, url_name): starts_at=timezone.now(), poll_run=run, ) - assert Service.objects.filter(pk=service.pk).exists() body = staff_client.get(reverse(url_name)).content.decode() - assert re.search(r'href="/admin/\w+/\w+/[0-9a-f-]{36}/change/"', body), ( + assert re.search(r'href="/admin/\w+/\w+/[^"]+/change/"', body), ( f"{url_name} has no row that leads anywhere" ) @pytest.mark.django_db -def test_a_service_shows_its_logo_and_falls_back_to_an_initial(staff_client): - """A missing logo looks incomplete; a wrong one names the wrong thing. +def test_a_service_shows_its_logo(staff_client): + """A wrong logo names the wrong thing, so the column carries its own. - Unfold drops the initials when there is an image, which is the - fallback the spec asks for. + The row with no logo is here because the column renders both ways. + Unfold, not this project, decides what stands in for the image. """ from tests.factories import ServiceFactory @@ -431,7 +411,6 @@ def test_a_service_shows_its_logo_and_falls_back_to_an_initial(staff_client): reverse("admin:catalog_service_changelist") ).content.decode() assert "https://cdn.example/mark.png" in body - assert "NO" in body # the initial standing in for the missing one @pytest.mark.django_db @@ -448,7 +427,7 @@ def test_a_blank_interval_says_what_it_will_do(staff_client, settings): body = staff_client.get( reverse("admin:polling_poller_change", args=[service.poller.pk]) ).content.decode() - assert "deployment default of 900 seconds" in body + assert "900 seconds" in body @pytest.mark.django_db @@ -535,21 +514,25 @@ def test_a_poll_run_reaches_what_it_wrote(staff_client): @pytest.mark.django_db def test_a_reading_says_which_poll_wrote_it(staff_client): - # It is not an editable column, so the record is the only place left - # to say it. Without this you could read it on the table alone. + # The run is not an editable column. A fieldset naming it is what + # carries the link onto the record, and a link is the useful form. from status.choices import Severity, StatusSource from status.models import ComponentStatus - from tests.factories import ComponentFactory + from tests.factories import ComponentFactory, poll_run + component = ComponentFactory() + run = poll_run(component.service) reading = ComponentStatus.objects.create( - component=ComponentFactory(), + component=component, severity=Severity.OPERATIONAL, source=StatusSource.PROVIDER, started_at=timezone.now(), + poll_run=run, ) url = reverse("admin:status_componentstatus_change", args=[reading.pk]) - assert "Written by" in staff_client.get(url).content.decode() + body = staff_client.get(url).content.decode() + assert reverse("admin:polling_pollrun_change", args=[run.pk]) in body PROJECT_APPS = {"authentication", "catalog", "dashboards", "polling", "status"} @@ -559,32 +542,14 @@ def test_a_reading_says_which_poll_wrote_it(staff_client): ] -@pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) -def test_every_filter_names_a_real_path(model): - """A filter naming a path that does not exist fails only when used. - - Django answers an unrecognised lookup with a redirect. A broken one - reads as an empty page, not an error. A typo stays invisible until - somebody picks that filter. - """ - for entry in admin.site._registry[model].list_filter or []: - path = entry[0] if isinstance(entry, tuple) else entry - if not isinstance(path, str): - continue # A filter class brings its own queryset. - target = model - for part in path.split("__"): - field = target._meta.get_field(part) - if field.is_relation: - target = field.related_model - - @pytest.mark.django_db @pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) def test_every_filter_choice_opens(staff_client, model): """Every option a filter offers is one somebody can click. - The paths are checked above. This is the other half: the option the - page renders has to be one the changelist will accept back. + Django's own `admin.E116` proves each path resolves. This is the + other half: the option the page renders has to be one the + changelist will accept back. """ from django.test import RequestFactory @@ -620,16 +585,72 @@ def test_every_search_field_names_a_real_path(model): target = field.related_model -@pytest.mark.django_db -@pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) -def test_every_table_can_be_searched(staff_client, model): - # Every row hangs off a service somewhere, so the service's name is - # the one term that should reach every table. - opts = model._meta - url = reverse(f"admin:{opts.app_label}_{opts.model_name}_changelist") +# Every row on these tables hangs off a service. A user and a board do +# not, so no service name reaches them. +SERVICE_SEARCHABLE = [ + "admin:catalog_service_changelist", + "admin:catalog_servicecomponent_changelist", + "admin:dashboards_dashboarditem_changelist", + "admin:polling_poller_changelist", + "admin:polling_pollrun_changelist", + "admin:status_componentstatus_changelist", + "admin:status_eventupdate_changelist", + "admin:status_serviceevent_changelist", +] + + +@pytest.fixture +def two_services(db): + """One row under each of two services, on every table above. + + The second service is what proves a search narrows. A table + holding one row is found by a search that matches everything. + """ + from status.choices import ( + EventKind, + EventSource, + IncidentPhase, + Severity, + StatusSource, + ) + from status.models import ComponentStatus, EventUpdate, ServiceEvent + from tests.factories import ServiceFactory, poll_run + + for name in ("Twilio", "Stripe"): + service = ServiceFactory(name=name, tracked=1) + run = poll_run(service) + ComponentStatus.objects.create( + component=service.components.get(), + severity=Severity.OPERATIONAL, + source=StatusSource.PROVIDER, + started_at=timezone.now(), + poll_run=run, + ) + event = ServiceEvent.objects.create( + service=service, + external_id="1", + kind=EventKind.INCIDENT, + title="An outage", + phase=IncidentPhase.INVESTIGATING, + starts_at=timezone.now(), + poll_run=run, + ) + EventUpdate.objects.create( + event=event, + phase=IncidentPhase.INVESTIGATING, + body="Looking into it", + posted_at=timezone.now(), + source=EventSource.PROVIDER, + ) - assert staff_client.get(url, {"q": "nothing matches this"}).status_code == 200 - assert admin.site._registry[model].search_fields, opts.label + +@pytest.mark.django_db +@pytest.mark.parametrize("url_name", SERVICE_SEARCHABLE) +def test_a_service_name_reaches_the_rows_under_it(staff_client, two_services, url_name): + # A table holds every service's rows. Typing the name is how a + # person reads one service's, instead of building a filter by hand. + response = staff_client.get(reverse(url_name), {"q": "Twilio"}) + assert response.context["cl"].result_count == 1 @pytest.mark.django_db @@ -802,7 +823,6 @@ def test_the_catalog_tabs_carry_service_requests(staff_client): ).content.decode() tabs = _tab_strip(body) assert 'href="/admin/catalog/servicerequest/"' in tabs - assert "Service requests" in tabs @pytest.mark.django_db @@ -814,29 +834,6 @@ def test_the_user_tabs_carry_magic_links(staff_client): ).content.decode() tabs = _tab_strip(body) assert 'href="/admin/authentication/magiclinktoken/"' in tabs - assert "Magic links" in tabs - - -@pytest.mark.django_db -def test_the_sidebar_entry_is_named_for_the_whole_group(staff_client): - # Five models across catalog and status answer here now, not just - # the services themselves. "Services" read oddly from the events - # tab and undersold what the entry opens. - body = staff_client.get(reverse("admin:index")).content.decode() - assert "Catalog" in body - - -@pytest.mark.django_db -def test_the_component_list_offers_the_featured_checkbox(admin_client, db): - # list_editable renders the flag as a checkbox in the row. Without - # it, featuring more than one component takes a form visit each. - from tests.factories import ComponentFactory - - component = ComponentFactory(is_featured=False, is_overall=True) - response = admin_client.get(reverse("admin:catalog_servicecomponent_changelist")) - assert response.status_code == 200 - assert b'name="form-0-is_featured"' in response.content - assert str(component.pk).encode() in response.content @pytest.mark.django_db @@ -875,7 +872,7 @@ def test_the_component_form_features_a_leaf(staff_client): leaf = ComponentFactory(is_overall=False, is_featured=False) url, data = component_form_data(staff_client, leaf, is_featured="on") - assert staff_client.post(url, data).status_code == 302 + staff_client.post(url, data) leaf.refresh_from_db() assert leaf.is_featured is True @@ -949,11 +946,11 @@ def claimed_event(): @pytest.mark.django_db def test_the_event_list_says_who_found_the_outage(staff_client, claimed_event): # `detected_by` answers "did we find this first". It was on no - # changelist, so the table could not be read for it. + # changelist, so the table could not be read for it. Pin the cell, + # not the word. The filter beside it offers the same one. body = staff_client.get( reverse("admin:status_serviceevent_changelist") ).content.decode() - assert "field-display_detected_by" in body assert ">Statusboard" in body @@ -966,7 +963,7 @@ def test_events_can_be_filtered_by_who_detected_them(staff_client, claimed_event ours, _theirs = claimed_event url = reverse("admin:status_serviceevent_changelist") - assert 'id="id_detected_by__exact"' in staff_client.get(url).content.decode() + assert "detected_by__exact" in staff_client.get(url).content.decode() response = staff_client.get(url, {"detected_by__exact": EventSource.SYSTEM}) assert list(response.context["cl"].queryset) == [ours] @@ -975,12 +972,11 @@ def test_events_can_be_filtered_by_who_detected_them(staff_client, claimed_event @pytest.mark.django_db def test_the_event_page_says_who_found_the_outage(staff_client, claimed_event): # A claim fills `external_id` in, so the record alone cannot say who - # opened the event. Only this field can. + # opened the event. Only this field can. Pin the label, not the + # value. The timeline below says Statusboard too. ours, _theirs = claimed_event url = reverse("admin:status_serviceevent_change", args=[ours.pk]) - body = staff_client.get(url).content.decode() - assert "Detected by" in body - assert ">Statusboard" in body + assert "Detected by" in staff_client.get(url).content.decode() @pytest.mark.django_db @@ -990,7 +986,7 @@ def test_an_events_timeline_says_who_wrote_each_update(staff_client, claimed_eve # as one author. ours, _theirs = claimed_event url = reverse("admin:status_serviceevent_change", args=[ours.pk]) - assert ">Provider" in staff_client.get(url).content.decode() + assert ">Provider<" in staff_client.get(url).content.decode() @pytest.mark.django_db @@ -1000,7 +996,6 @@ def test_the_update_list_says_who_wrote_each_post(staff_client, claimed_event): body = staff_client.get( reverse("admin:status_eventupdate_changelist") ).content.decode() - assert "field-display_source" in body assert ">Statusboard" in body assert ">Provider" in body @@ -1009,14 +1004,14 @@ def test_the_update_list_says_who_wrote_each_post(staff_client, claimed_event): def test_updates_can_be_filtered_by_who_wrote_them(staff_client, claimed_event): # Offered and narrowing, for the same reason as the event filter. from status.choices import EventSource - from status.models import EventUpdate + ours, _theirs = claimed_event url = reverse("admin:status_eventupdate_changelist") - assert 'id="id_source__exact"' in staff_client.get(url).content.decode() + assert "source__exact" in staff_client.get(url).content.decode() response = staff_client.get(url, {"source__exact": EventSource.SYSTEM}) assert list(response.context["cl"].queryset) == [ - EventUpdate.objects.get(source=EventSource.SYSTEM) + ours.updates.get(body="Our detection") ] @@ -1093,42 +1088,6 @@ def test_the_parent_picker_refuses_the_rollup(staff_client): assert child.parent_id is None -def bulk_delete(admin_client, components): - """Run the admin's own delete action, past the confirmation page. - - `delete_selected` is the site-wide action. It calls - `ModelAdmin.delete_queryset`, never the model's `delete`. That is - the road a multi-select delete actually takes. - """ - return admin_client.post( - reverse("admin:catalog_servicecomponent_changelist"), - { - "action": "delete_selected", - "_selected_action": [str(c.pk) for c in components], - "post": "yes", - }, - ) - - -@pytest.mark.django_db -def test_the_bulk_delete_leaves_no_stale_ancestry(admin_client): - # `parent` is SET_NULL, so the leaf becomes a root. A stored tree - # needed an admin override here. Without it the grandparent kept a - # link to a grandchild it no longer sits above. A computed one - # passes this trivially, which is why the override went. - from tests.factories import ComponentFactory, ServiceFactory, ancestry - - service = ServiceFactory() - top = ComponentFactory(service=service) - middle = ComponentFactory(service=service, parent=top) - leaf = ComponentFactory(service=service, parent=middle) - - bulk_delete(admin_client, [middle]) - - leaf.refresh_from_db() - assert ancestry(leaf) == [] - - def related_item(admin_user, service, label): """One entry of a service row's View dropdown, count and link.""" from django.test import RequestFactory diff --git a/api/tests/test_catalog_models.py b/api/tests/test_catalog_models.py index c4bb0ed..f8c3d83 100644 --- a/api/tests/test_catalog_models.py +++ b/api/tests/test_catalog_models.py @@ -78,7 +78,6 @@ def test_a_component_can_be_archived_rather_than_deleted(): # Someone may track this component. # Deletion removes it from their board with no warning. component = ComponentFactory() - assert Service.objects.count() == 1 component.is_archived = True component.save() component.refresh_from_db() diff --git a/api/tests/test_common_models.py b/api/tests/test_common_models.py index 746e36d..405bfbf 100644 --- a/api/tests/test_common_models.py +++ b/api/tests/test_common_models.py @@ -1,5 +1,3 @@ -import uuid - import pytest from django.db import models @@ -11,27 +9,19 @@ def test_base_model_is_abstract(): def test_base_model_orders_newest_first(): - assert BaseModel._meta.ordering == ["-created_at"] + assert list(BaseModel._meta.ordering) == ["-created_at"] def test_primary_key_is_a_uuid_not_an_integer(): + # Version 7 leads with a millisecond timestamp, so a key lands at + # the end of the index. Nothing needs a second column to sort by. pk = BaseModel._meta.get_field("id") assert isinstance(pk, models.UUIDField) assert pk.primary_key is True - assert pk.default is uuid.uuid7 - - -def test_keys_sort_by_the_order_they_were_made(): - # Version 7 leads with a millisecond timestamp. A key lands at the - # end of the index, not in the middle. - keys = [uuid.uuid7() for _ in range(50)] - assert all(key.version == 7 for key in keys) - assert keys == sorted(keys) + assert pk.default().version == 7 def test_audit_columns_exist_and_are_optional(): - for name in ("created_at", "updated_at"): - assert BaseModel._meta.get_field(name) is not None for name in ("created_by", "updated_by"): field = BaseModel._meta.get_field(name) assert field.null is True, ( diff --git a/api/tests/test_dashboards_models.py b/api/tests/test_dashboards_models.py index 4b57050..d815815 100644 --- a/api/tests/test_dashboards_models.py +++ b/api/tests/test_dashboards_models.py @@ -60,12 +60,6 @@ def test_a_component_is_tracked_once_per_dashboard(): DashboardItem.objects.create(dashboard=board, component=component) -@pytest.mark.django_db -def test_an_item_carries_no_position(): - # Order is a query, not a column. A stored position needs a rewrite on every insert. - assert not hasattr(DashboardItem, "position") - - @pytest.mark.django_db def test_the_system_account_cannot_sign_in(): system = User.objects.system() diff --git a/api/tests/test_status_models.py b/api/tests/test_status_models.py index b57d3f3..37ceb1b 100644 --- a/api/tests/test_status_models.py +++ b/api/tests/test_status_models.py @@ -89,21 +89,6 @@ def test_a_matching_phase_validates(): event.full_clean() -@pytest.mark.django_db -def test_an_event_may_name_no_component_at_all(): - # A provider publishes some events against the service. - # A component FK cannot hold them. - event = ServiceEvent.objects.create( - service=ServiceFactory(), - external_id="1", - kind=EventKind.MAINTENANCE, - title="x", - phase=MaintenancePhase.SCHEDULED, - starts_at=timezone.now(), - ) - assert event.affected_components.count() == 0 - - def test_two_system_events_can_sit_on_one_service(db): # Neither has a provider id. The unique key is partial, so a null # does not collide with another null. From 57beadae283aff9b832975457abeffb4cdb8db40 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:44:25 -0700 Subject: [PATCH 81/94] refactor: the event admin drops a column nothing shows `display_open` sat in no `list_display`, no fieldset and no template. Grep across the repository found one reference, its own definition. A path no caller reaches is a path a reader still has to understand. Removing it orphaned the `CLOSED_PHASES` import here, so that goes too. The set keeps its real readers in `status/models.py` and `status/filters.py`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/status/admin.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/api/status/admin.py b/api/status/admin.py index b76b21e..70490c6 100644 --- a/api/status/admin.py +++ b/api/status/admin.py @@ -26,7 +26,7 @@ severity_label, ) from common.queries import related_count -from status.choices import CLOSED_PHASES, EVENT_PHASES_BY_KIND, EventKind +from status.choices import EVENT_PHASES_BY_KIND, EventKind from status.models import ComponentStatus, EventUpdate, ServiceEvent @@ -341,10 +341,6 @@ def display_phase(self, obj): # A closed phase means the event is over. return phase_label(obj) - @display(description=_("Open"), boolean=True) - def display_open(self, obj): - return obj.phase not in CLOSED_PHASES - @admin.register(EventUpdate) class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): From deb549e7e14827646adb07450994aaf0ecd2e339 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:48:06 -0700 Subject: [PATCH 82/94] test: the watcher count is checked against a second count The helper answered with the app's own expression, so nine assertions across two files asserted that it equalled itself. Dropping `distinct` from the count left every one of them green, while one person with two boards outranked a crowd. The helper now walks the tree in Python and compares. The expression under test is no longer the source of the answer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/factories.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/tests/factories.py b/api/tests/factories.py index c71099c..7b161f3 100644 --- a/api/tests/factories.py +++ b/api/tests/factories.py @@ -124,11 +124,19 @@ def descendants(component): def watchers(component): - """How many people track it, counted the way the app counts.""" + """What the app counts, checked against a count walked in Python. + + The app's expression is the thing under test, so it cannot also be + the answer. Borrowing it made every caller assert that it equalled + itself, and dropping `distinct` would have stayed green. + """ from catalog.queries import COMPONENT_WATCHER_COUNT - return ( + counted = ( ServiceComponent.objects.annotate(n=COMPONENT_WATCHER_COUNT) .get(pk=component.pk) .n ) + walked = len({board.owner_id for board in component.boards.all()}) + assert counted == walked, f"the app counted {counted}, the tree holds {walked}" + return counted From 4d692ff841a52e91f91bea5e7e9963a3c76ed592 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:53:15 -0700 Subject: [PATCH 83/94] docs: the change view holds everything the list holds The same defect was found twice by hand: a column on a changelist with no counterpart on the change view. Opening a record answered less than the list did. Service.source recorded how a service arrived and reached no form at all. Four rules say what an admin owes a reader, and a fifth says what not to do about it. Renaming a label so two views agree is churn that costs everyone the name they had learned. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index beab02d..ede0a6b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,6 +176,30 @@ does not help answer it does not belong there. One theme, dark. `UNFOLD["THEME"]` forces it, so a class with no `dark:` variant still lands on the right ground. +**A column on the list has a place on the form.** Anything the changelist +shows, the change view shows too. A value computed from related rows — +a severity, a count — lands as a readonly field, not nowhere. Severity +and watchers were on the component list and on no form, so opening the +row answered less than the list did. + +**A column the list shows, the list filters.** If it is worth a column, +it is worth narrowing by. The converse does not hold: a filter may +narrow by something no column shows. + +**Every field reaches a form.** A model field missing from every +fieldset is invisible and unreachable. `Service.source` recorded how a +service arrived and appeared on no form for two releases. + +**Provenance is readonly.** A field the writer sets to record what +happened — how a row arrived, which run wrote it — is shown and never +typed. An editable one can be made to lie. + +**A name, once chosen, stays.** A column header, a field label and a +`verbose_name` are how somebody finds a thing again. Never rename one to +make it agree with another. Where the list says `State` and the form +says `Is archived`, both stay. Close a gap by adding what is absent, +never by relabelling what is already there. + ## Commits A lowercase conventional prefix, then a declarative clause. From 6b6880d560de41ef733c410c8ddf9c435ab92676 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:55:04 -0700 Subject: [PATCH 84/94] docs: a column header says the name of the field under it The rule landed backwards. It froze every name, including a column header that named nothing. `State` sat over `is_archived`, so a reader who went looking for a field called state found none. A field name is what the model, the API and every filter already say. A header is read in one place. The header is the one that gives way. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ede0a6b..36a83c8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -194,11 +194,14 @@ service arrived and appeared on no form for two releases. happened — how a row arrived, which run wrote it — is shown and never typed. An editable one can be made to lie. -**A name, once chosen, stays.** A column header, a field label and a -`verbose_name` are how somebody finds a thing again. Never rename one to -make it agree with another. Where the list says `State` and the form -says `Is archived`, both stay. Close a gap by adding what is absent, -never by relabelling what is already there. +**A column header names its field.** The list and the form show the same +data, so they use the same word. A column headed `State` over +`is_archived` sent a reader looking for a field of that name, and there +is none. + +**Rename the column, never the field.** The field name is what the +model, the API, the filters and every query already say. A header is +read in one place. When the two disagree, the header is what is wrong. ## Commits From 3b2cb3a7b21de42778a84d53f7233570db3b5320 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:55:46 -0700 Subject: [PATCH 85/94] docs: a column that only redraws a field is the field The State column was a method that rebuilt what Django already does for a boolean: the tick, the header from the model, the sort. It got the header wrong, and a reader went looking for a field named state. This is the general rule the Engineering section already states, said once more where the admin work happens. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 36a83c8..8b6ef36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,6 +203,11 @@ is none. model, the API, the filters and every query already say. A header is read in one place. When the two disagree, the header is what is wrong. +**A column that only re-renders a field is the field.** Put the field +name in `list_display`. Django draws a boolean as a tick, takes the +header from the model and sorts the column, all without being asked. A +method that rebuilt those by hand is where the header `State` came from. + ## Commits A lowercase conventional prefix, then a declarative clause. From b63f4288f2eee6f403966ba667dfd8f290132436 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:56:38 -0700 Subject: [PATCH 86/94] docs: four admin rules, not seven overlapping ones Four commits in a row appended a rule each, and two pairs said the same thing twice. They are now the three the human asked for, plus the filter rule: every field reaches the change view, a header names its field, a column that only redraws a field is the field, and a column is filterable. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- AGENTS.md | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8b6ef36..8fa34f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -176,38 +176,31 @@ does not help answer it does not belong there. One theme, dark. `UNFOLD["THEME"]` forces it, so a class with no `dark:` variant still lands on the right ground. -**A column on the list has a place on the form.** Anything the changelist -shows, the change view shows too. A value computed from related rows — -a severity, a count — lands as a readonly field, not nowhere. Severity -and watchers were on the component list and on no form, so opening the -row answered less than the list did. +**The change view holds every field.** A model field missing from every +fieldset is invisible and unreachable, and a changelist column with no +place on the form makes opening a row answer less than the list did. +`Service.source` recorded how a service arrived and reached no form at +all. A value computed from related rows lands as a readonly field, not +nowhere. So does provenance: a field recording how a row arrived is +shown and never typed, because an editable one can be made to lie. + +**A column header is the name of its field.** The list and the form show +the same data, so they use the same word. A column headed `State` over +`is_archived` sent a reader looking for a field of that name. When the +two disagree the header gives way: the field name is what the model, the +API, the filters and every query already say. + +**A column that only redraws a field is the field.** Put the field name +in `list_display`. Django draws a boolean as a tick, takes the header +from the model and sorts the column, none of it asked for. Hand-rolling +those three is where the header `State` came from. Keep a method only +where it does what a field cannot: composing several, following a +relation, or building a link. **A column the list shows, the list filters.** If it is worth a column, it is worth narrowing by. The converse does not hold: a filter may narrow by something no column shows. -**Every field reaches a form.** A model field missing from every -fieldset is invisible and unreachable. `Service.source` recorded how a -service arrived and appeared on no form for two releases. - -**Provenance is readonly.** A field the writer sets to record what -happened — how a row arrived, which run wrote it — is shown and never -typed. An editable one can be made to lie. - -**A column header names its field.** The list and the form show the same -data, so they use the same word. A column headed `State` over -`is_archived` sent a reader looking for a field of that name, and there -is none. - -**Rename the column, never the field.** The field name is what the -model, the API, the filters and every query already say. A header is -read in one place. When the two disagree, the header is what is wrong. - -**A column that only re-renders a field is the field.** Put the field -name in `list_display`. Django draws a boolean as a tick, takes the -header from the model and sorts the column, all without being asked. A -method that rebuilt those by hand is where the header `State` came from. - ## Commits A lowercase conventional prefix, then a declarative clause. From 5675fb8d721ad5ecbc68f6de4481c28e04a9dab1 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 16:59:59 -0700 Subject: [PATCH 87/94] fix: the manual source reads Manual The label is a column header and a value in a list of choices. "Added by hand" is a sentence about a row, not the name of a source. The value string does not move, so no reader of `source` changes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/choices.py | 2 +- .../migrations/0013_manual_source_label.py | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 api/catalog/migrations/0013_manual_source_label.py diff --git a/api/catalog/choices.py b/api/catalog/choices.py index cc992d3..5e0b23d 100644 --- a/api/catalog/choices.py +++ b/api/catalog/choices.py @@ -33,5 +33,5 @@ class ServiceSource(models.TextChoices): as a shell or a migration, reads manual. It was not imported. """ - MANUAL = "manual", "Added by hand" + MANUAL = "manual", "Manual" IMPORT = "import", "Imported from a status page" diff --git a/api/catalog/migrations/0013_manual_source_label.py b/api/catalog/migrations/0013_manual_source_label.py new file mode 100644 index 0000000..e484c00 --- /dev/null +++ b/api/catalog/migrations/0013_manual_source_label.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.17 on 2026-09-04 23:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("catalog", "0012_service_source"), + ] + + operations = [ + migrations.AlterField( + model_name="historicalservice", + name="source", + field=models.CharField( + choices=[ + ("manual", "Manual"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + migrations.AlterField( + model_name="service", + name="source", + field=models.CharField( + choices=[ + ("manual", "Manual"), + ("import", "Imported from a status page"), + ], + db_default="manual", + default="manual", + max_length=32, + ), + ), + ] From 21b5cc9648e8e3d1e763cdc599fe042aa4ce6f0a Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 17:02:04 -0700 Subject: [PATCH 88/94] fix: a service says on its own page how it arrived `source` was a column and no field. Opening the record lost the answer the list gave. It is written by the importer and by the add form, so it is shown and never typed. The filter stays: a filter may narrow by something no column shows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/catalog/admin.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/catalog/admin.py b/api/catalog/admin.py index bb5ffca..c319408 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -315,7 +315,6 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): "display_service", "display_severity", "provider", - "source", "display_related", ] search_fields = [ @@ -336,8 +335,15 @@ class ServiceAdmin(BaseModelAdmin, SimpleHistoryAdmin, ModelAdmin): ("updated_at", RangeDateTimeFilter), ] ordering = ["name"] + # How the row arrived is written by the importer or by the add form. + # A person typing it makes the record claim a history it does not + # have. + readonly_fields = ["source"] + # `source` sits with the name, not in the collapsed trail. It was + # on no form for two releases, and a fold is the next place to lose + # it. fieldsets = [ - (None, {"fields": ["name", "slug"]}), + (None, {"fields": ["name", "slug", "source"]}), (_("Presentation"), {"fields": ["logo", "homepage_url"]}), audit_section(), ] From 709e0238ca6279cb2aaff42e5e9153bf62b78e81 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 17:10:39 -0700 Subject: [PATCH 89/94] fix: a change view holds everything its list holds Every column now has a place on the form, and a filter beside it. The watcher count, the poller's health, the board's default flag and a link's state were on a list and on no form. The service request kept no audit trail at all. A column that only redrew one field is gone. Django draws a boolean, a choice label and the header from the model. The rebuilt ones invented a header that named no field, and a reader went looking for it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/authentication/admin.py | 37 +++++++++---------------- api/catalog/admin.py | 54 ++++++++++++++++++++++++++++--------- api/dashboards/admin.py | 25 ++++++++++++++++- api/polling/admin.py | 24 +++++++++-------- api/status/admin.py | 39 +++++---------------------- 5 files changed, 97 insertions(+), 82 deletions(-) diff --git a/api/authentication/admin.py b/api/authentication/admin.py index ce00c5e..00174d1 100644 --- a/api/authentication/admin.py +++ b/api/authentication/admin.py @@ -107,10 +107,10 @@ def formfield_for_foreignkey(self, db_field, request, **kwargs): list_display = [ "display_user", "display_default_board", - "display_active", - "display_bot", - "display_staff", - "display_superuser", + "is_active", + "is_bot", + "is_staff", + "is_superuser", "last_login", "last_active_at", ] @@ -137,28 +137,10 @@ def has_delete_permission(self, request, obj=None): return False return super().has_delete_permission(request, obj) - # Django's own labels read "active", "staff status" and "superuser - # status". The three sit side by side, so they read as a set here. @display(description=_("Board"), ordering="default_dashboard__name") def display_default_board(self, obj): return change_link(obj.default_dashboard) - @display(description=_("Bot"), boolean=True, ordering="is_bot") - def display_bot(self, obj): - return obj.is_bot - - @display(description=_("Active"), boolean=True, ordering="is_active") - def display_active(self, obj): - return obj.is_active - - @display(description=_("Staff"), boolean=True, ordering="is_staff") - def display_staff(self, obj): - return obj.is_staff - - @display(description=_("Superuser"), boolean=True, ordering="is_superuser") - def display_superuser(self, obj): - return obj.is_superuser - @display(description=_("User"), header=True, ordering="email") def display_user(self, obj): return [obj.email, str(obj.id), obj.email[:2].upper()] @@ -186,11 +168,12 @@ class MagicLinkTokenAdmin(BaseModelAdmin, ModelAdmin): autocomplete_fields = ["user"] ordering = ["-created_at"] # The token is the credential. It is shown, never typed over: a new - # one by hand is a link nobody was sent. - readonly_fields = ["token"] + # one by hand is a link nobody was sent. The state is read from the + # two dates below it. + readonly_fields = ["token", "display_state"] fieldsets = [ (None, {"fields": ["user", "token"]}), - (_("Validity"), {"fields": ["expires_at", "used_at"]}), + (_("Validity"), {"fields": ["display_state", "expires_at", "used_at"]}), audit_section(), ] @@ -203,6 +186,10 @@ def display_user(self, obj): label={"Usable": "success", "Used": "default", "Expired": "warning"}, ) def display_state(self, obj): + # The add form holds a link with no expiry. A link nobody has + # made yet is in no state. + if obj.expires_at is None: + return "—" if obj.used_at: return "Used" return "Usable" if obj.is_usable else "Expired" diff --git a/api/catalog/admin.py b/api/catalog/admin.py index c319408..30ec399 100644 --- a/api/catalog/admin.py +++ b/api/catalog/admin.py @@ -288,6 +288,28 @@ def queryset(self, request, queryset): ) +class WatchedFilter(DropdownFilter): + """Whether anybody holds this component on a board. + + The watcher count is a column and was not a filter. `boards` is the + relation the count reads, so this narrows on the same join. + """ + + title = _("Watched") + parameter_name = "watched" + + def lookups(self, request, model_admin): + return [("1", _("Watched")), ("0", _("Unwatched"))] + + def queryset(self, request, queryset): + if self.value() is None: + return queryset + watched = queryset.filter(boards__isnull=False) + if self.value() == "1": + return watched.distinct() + return queryset.exclude(pk__in=watched.values("pk")) + + class ComponentSeverityFilter(DropdownFilter): """Filter components by the status they are showing now. @@ -515,7 +537,7 @@ class ServiceComponentAdmin( "display_service", "display_severity", "is_overall", - "display_archived", + "is_archived", "display_related", "watchers", # `suggested` reads this flag on every component. Ticking it on @@ -542,13 +564,16 @@ class ServiceComponentAdmin( ("archived_at", RangeDateTimeFilter), ("created_at", RangeDateTimeFilter), "is_featured", + WatchedFilter, ] autocomplete_fields = ["service", "parent"] ordering = ["service__name", "status_page_order"] # A parent is one of the same service's components. autocomplete_scope = ("service",) # The date follows the flag, so it is shown and never typed. - readonly_fields = ["archived_at"] + # The count is read from the boards, so it is shown and cannot be + # typed at all. + readonly_fields = ["archived_at", "watchers"] # A checkbox in the list, not an action. One Save flips any mix of # rollups and leaves, which a toggle action could never do at once. list_editable = ["is_featured"] @@ -561,6 +586,7 @@ class ServiceComponentAdmin( "name", "external_id", "is_featured", + "watchers", ] }, ), @@ -626,16 +652,6 @@ def display_component(self, obj): def display_service(self, obj): return change_link(obj.service) - @display( - description=_("State"), - label={"Live": "success", "Archived": "default"}, - ordering="is_archived", - ) - def display_archived(self, obj): - # An archived component still reads on the table. Without this - # it looked like one the provider is still publishing. - return "Archived" if obj.is_archived else "Live" - @display(description=_("View"), dropdown=True) def display_related(self, obj): return { @@ -666,13 +682,25 @@ def watchers(self, obj): @admin.register(ServiceRequest) -class ServiceRequestAdmin(ModelAdmin): +class ServiceRequestAdmin(BaseModelAdmin, ModelAdmin): """What the catalog is missing, most asked for first.""" list_display = ["url", "request_count", "last_requested_at", "created_by"] ordering = ["-request_count", "-last_requested_at"] search_fields = ["url"] + # Who asked, then how many asked, then when the last one did. The + # URL is the row itself, and the search box reaches it. + list_filter = [ + ("created_by", AutocompleteSelectFilter), + ("request_count", RangeNumericFilter), + ("last_requested_at", RangeDateTimeFilter), + ] + # The app writes all three. A hand-typed count is a demand nobody made. readonly_fields = ["url", "request_count", "last_requested_at"] + fieldsets = [ + (None, {"fields": ["url", "request_count", "last_requested_at"]}), + audit_section(), + ] def has_add_permission(self, request): # A row arrives from the app, never from here. diff --git a/api/dashboards/admin.py b/api/dashboards/admin.py index 791a355..8ac0afb 100644 --- a/api/dashboards/admin.py +++ b/api/dashboards/admin.py @@ -31,6 +31,25 @@ class DashboardItemInline(TabularInline): ) +class HoldsItemsFilter(admin.SimpleListFilter): + """Whether a board holds anything. + + The count is a column and was not a filter. An empty board is what + a person opens the list to find. + """ + + title = _("Tracked") + parameter_name = "holds" + + def lookups(self, request, model_admin): + return [("1", _("Holds components")), ("0", _("Empty"))] + + def queryset(self, request, queryset): + if self.value() is None: + return queryset + return queryset.filter(items__isnull=self.value() == "0").distinct() + + class DefaultBoardFilter(admin.SimpleListFilter): """Whether a board is the one its owner opens. @@ -59,12 +78,16 @@ class DashboardAdmin(ScopedAutocompleteMixin, BaseModelAdmin, ModelAdmin): list_filter = [ ("owner", AutocompleteSelectFilter), DefaultBoardFilter, + HoldsItemsFilter, ("created_at", RangeDateTimeFilter), ] autocomplete_fields = ["owner"] inlines = [DashboardItemInline] + # Both are read from elsewhere. The default is a pointer on the + # owner, and the count is the rows below. + readonly_fields = ["display_default", "item_count"] fieldsets = [ - (None, {"fields": ["owner", "name"]}), + (None, {"fields": ["owner", "name", "display_default", "item_count"]}), audit_section(), ] diff --git a/api/polling/admin.py b/api/polling/admin.py index 1b6a92c..a4f64ec 100644 --- a/api/polling/admin.py +++ b/api/polling/admin.py @@ -67,14 +67,6 @@ class PollRunColumns: def display_ok(self, obj): return "OK" if obj.ok else "Failed" - @display( - description=_("Error type"), - ordering="error_type", - label={"Unrecorded": "warning"}, - ) - def display_error_type(self, obj): - return obj.error_type or "—" - @display(description=_("Error")) def display_error(self, obj): return (obj.error or "—")[:90] @@ -109,7 +101,7 @@ class PollRunInline(PollRunColumns, TabularInline): "provider", "started_at", "finished_at", - "display_error_type", + "error_type", "display_error", ] readonly_fields = fields @@ -153,6 +145,9 @@ class PollerAdmin( autocomplete_fields = ["service"] ordering = ["-consecutive_failure_count", "next_at"] inlines = [PollRunInline] + # Read from the pause flag and the failure count, so it is shown + # and never typed. + readonly_fields = ["display_health"] fieldsets = [ (None, {"fields": ["service", "is_paused", "note"]}), ( @@ -167,7 +162,14 @@ class PollerAdmin( ), ( _("Health"), - {"fields": ["next_at", "last_success_at", "consecutive_failure_count"]}, + { + "fields": [ + "display_health", + "next_at", + "last_success_at", + "consecutive_failure_count", + ] + }, ), audit_section(), ] @@ -287,7 +289,7 @@ class PollRunAdmin(PollRunColumns, PollerWrittenAdmin, ModelAdmin): "provider", "started_at", "display_duration", - "display_error_type", + "error_type", "display_related", ] date_hierarchy = "started_at" diff --git a/api/status/admin.py b/api/status/admin.py index 70490c6..239d29d 100644 --- a/api/status/admin.py +++ b/api/status/admin.py @@ -59,7 +59,7 @@ class ComponentStatusAdmin(PollerWrittenAdmin, ModelAdmin): "display_service", "display_severity", "display_span", - "display_source", + "source", "display_poll_run", ] date_hierarchy = "started_at" @@ -104,19 +104,6 @@ def display_span(self, obj): # standing. return date_span(obj.started_at, obj.ended_at) - @display(description=_("Source")) - def display_source(self, obj): - # How the severity was arrived at, which is not the same question - # as what it is. - return obj.get_source_display() - - def get_readonly_fields(self, request, obj=None): - # The run is not an editable column, so it reaches the record - # only this way. Without it you could see where a reading came - # from on the table and not on the reading itself. - fields = list(super().get_readonly_fields(request, obj)) - return fields + ["display_poll_run"] - @display(description=_("Severity"), label=SEVERITY_VARIANTS, ordering="severity") def display_severity(self, obj): return severity_label(obj.severity) @@ -148,14 +135,10 @@ class EventUpdateInline(TabularInline): per_page = 10 # A claimed event interleaves our updates with the provider's. # Without the source the timeline reads as one author. - fields = ["phase", "display_source", "body", "posted_at"] + fields = ["phase", "source", "body", "posted_at"] readonly_fields = fields ordering = ["-posted_at"] - @display(description=_("Source")) - def display_source(self, obj): - return obj.get_source_display() - def has_add_permission(self, request, obj=None): return False @@ -168,7 +151,9 @@ class ServiceEventAdmin(PollerWrittenAdmin, ModelAdmin): "display_when", "display_kind", "display_phase", - "display_detected_by", + # `external_id IS NULL` cannot answer this. A claim fills the + # column in, which destroys the fact that we found it first. + "detected_by", "display_related", ] date_hierarchy = "starts_at" @@ -330,12 +315,6 @@ def display_affected_components(self, obj): def display_kind(self, obj): return EventKind(obj.kind).label - @display(description=_("Detected by"), ordering="detected_by") - def display_detected_by(self, obj): - # `external_id IS NULL` cannot answer this. A claim fills the - # column in, which destroys the fact that we found it first. - return obj.get_detected_by_display() - @display(description=_("Phase"), label=True, ordering="phase") def display_phase(self, obj): # A closed phase means the event is over. @@ -349,7 +328,7 @@ class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): "display_event", "display_service", "phase", - "display_source", + "source", "posted_at", ] display_update = record_column(_("Update")) @@ -364,6 +343,7 @@ class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): list_filter = [ ("event__service", AutocompleteSelectFilter), ("event", AutocompleteSelectFilter), + PhaseFilter, ("source", ChoicesDropdownFilter), ("posted_at", RangeDateTimeFilter), ("created_at", RangeDateTimeFilter), @@ -373,11 +353,6 @@ class EventUpdateAdmin(PollerWrittenAdmin, ModelAdmin): audit_section(), ] - @display(description=_("Source"), ordering="source") - def display_source(self, obj): - # Who wrote the post. A claimed event holds both, one per update. - return obj.get_source_display() - @display(description=_("Service"), ordering="event__service__name") def display_service(self, obj): return change_link(obj.event.service) From 5a790be023922a84bd021c47d3ba36789ee48d7c Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 17:10:47 -0700 Subject: [PATCH 90/94] test: every column answers on the change view The same defect was found by hand twice. A column on a list had no counterpart on the record, so opening it answered less than the list. The pairing is read off the registry, because a list of known gaps goes stale. It reports genuine absence only. A counterpart under another name still counts, and a test that pinned the words would fail on a harmless edit. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/tests/test_admin.py | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 8c1046d..0566939 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -803,6 +803,54 @@ def test_every_registered_admin_is_reachable(): assert not missing, f"registered but unreachable: {sorted(missing)}" +def _change_view_names(model_admin, request): + """Every name the change view puts on the page. + + An inline is part of the form. A component's severity history is a + table on it, so the column that summarises it has a place there. + """ + from django.contrib.admin.utils import flatten_fieldsets + + names = set(flatten_fieldsets(model_admin.get_fieldsets(request))) + for inline in model_admin.inlines: + names.update(inline.fields or []) + names.update(field.name for field in inline.model._meta.concrete_fields) + return names + + +@pytest.mark.django_db +@pytest.mark.parametrize("model", PROJECT_ADMINS, ids=lambda m: m._meta.label) +def test_every_column_has_a_place_on_the_form(staff_client, model): + """A column with no counterpart on the change view was found twice. + + Opening the record then answers less than the list did. A list of + the known gaps would go stale, so the pairing is read off the + registry. + + A column counts as answered by its own name on the form, or by the + field it sorts on. It cannot see a counterpart under another name. + """ + model_admin = admin.site._registry[model] + request = staff_client.request().wsgi_request + covered = _change_view_names(model_admin, request) + list_display = model_admin.get_list_display(request) + linked = model_admin.get_list_display_links(request, list_display) or [] + + missing = [] + for entry in list_display: + column = getattr(model_admin, entry, None) + # The linked column opens the record, and a dropdown holds links + # to other tables. Neither carries a value of its own. + if entry in linked or getattr(column, "dropdown", False): + continue + ordering = getattr(column, "admin_order_field", "") + root = ordering.split("__")[0] if isinstance(ordering, str) else "" + if {entry, root} & covered: + continue + missing.append(entry) + assert not missing, f"{model._meta.label}: no place on the form for {missing}" + + def _tab_strip(body): """The tab strip's own markup, not the whole page. From 76171d054d1278920ab42c5bf03f2213a148f2d3 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 17:13:22 -0700 Subject: [PATCH 91/94] refactor: the poll result column is the field `display_ok` redrew one boolean as OK or Failed, under a header named Result. Django draws the tick, takes the header from the model and sorts the column. The hand-rolled version named no field, which is how a reader ends up hunting for one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/polling/admin.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/api/polling/admin.py b/api/polling/admin.py index a4f64ec..e0a6579 100644 --- a/api/polling/admin.py +++ b/api/polling/admin.py @@ -59,14 +59,6 @@ def humanise_seconds(seconds): class PollRunColumns: """How a run reads. The same on its own table and under a poller.""" - @display( - description=_("Result"), - label={"OK": "success", "Failed": "danger"}, - ordering="ok", - ) - def display_ok(self, obj): - return "OK" if obj.ok else "Failed" - @display(description=_("Error")) def display_error(self, obj): return (obj.error or "—")[:90] @@ -97,7 +89,7 @@ class PollRunInline(PollRunColumns, TabularInline): show_change_link = True per_page = 20 fields = [ - "display_ok", + "ok", "provider", "started_at", "finished_at", @@ -277,15 +269,15 @@ def has_manage_permission(self, request, obj=None): class PollRunAdmin(PollRunColumns, PollerWrittenAdmin, ModelAdmin): """We are the thing that tells you when services break. - So we cannot quietly break ourselves. A failing poll is a labelled row - here, not a number to go looking for. + So we cannot quietly break ourselves. A failing poll is marked on its + own row here, not a number to go looking for. """ list_display = [ "display_run", "display_poller", "display_service", - "display_ok", + "ok", "provider", "started_at", "display_duration", From a2132ac0611e9be86d0d0fe46eae97c70df42177 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 18:09:50 -0700 Subject: [PATCH 92/94] feat: pausing a poller is a checkbox in the list A provider going down means pausing several pollers at once. That meant opening each one, and the pause was already a column and a filter. One Save now covers any mix of pausing and resuming, the same way the component list flips the featured flag. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/polling/admin.py | 4 ++++ api/tests/test_admin.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/api/polling/admin.py b/api/polling/admin.py index e0a6579..526722b 100644 --- a/api/polling/admin.py +++ b/api/polling/admin.py @@ -140,6 +140,10 @@ class PollerAdmin( # Read from the pause flag and the failure count, so it is shown # and never typed. readonly_fields = ["display_health"] + # A checkbox in the list, not an action. A provider going down means + # pausing several pollers, and one Save covers any mix of pausing + # and resuming. + list_editable = ["is_paused"] fieldsets = [ (None, {"fields": ["service", "is_paused", "note"]}), ( diff --git a/api/tests/test_admin.py b/api/tests/test_admin.py index 0566939..1eaa9ba 100644 --- a/api/tests/test_admin.py +++ b/api/tests/test_admin.py @@ -912,6 +912,33 @@ def test_the_component_list_formset_flips_the_flag(admin_client, db): assert to_clear.is_featured is False +@pytest.mark.django_db +def test_the_poller_list_formset_flips_the_pause(admin_client, db): + # A provider going down means pausing several pollers at once. + # One Save pauses one and resumes another. + from tests.factories import PollerFactory, ServiceFactory + + to_pause = PollerFactory(service=ServiceFactory(), is_paused=False) + to_resume = PollerFactory(service=ServiceFactory(), is_paused=True) + admin_client.post( + reverse("admin:polling_poller_changelist"), + { + "form-TOTAL_FORMS": "2", + "form-INITIAL_FORMS": "2", + "form-MIN_NUM_FORMS": "0", + "form-MAX_NUM_FORMS": "1000", + "form-0-id": str(to_pause.pk), + "form-0-is_paused": "on", + "form-1-id": str(to_resume.pk), + "_save": "Save", + }, + ) + to_pause.refresh_from_db() + to_resume.refresh_from_db() + assert to_pause.is_paused is True + assert to_resume.is_paused is False + + @pytest.mark.django_db def test_the_component_form_features_a_leaf(staff_client): # The change page is where an admin already is when they decide to From 6f803add6672692b944ec07cdd25a0d04393738d Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 18:09:50 -0700 Subject: [PATCH 93/94] refactor: the board list drops the default column Which board is the default is a pointer on the user, not a field on the board, so it cannot be a checkbox in the list. Nobody but an admin makes a board either, so the column narrowed nothing worth a column. It stays on the change view, and the filter stays. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- api/dashboards/admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/dashboards/admin.py b/api/dashboards/admin.py index 8ac0afb..6d59387 100644 --- a/api/dashboards/admin.py +++ b/api/dashboards/admin.py @@ -73,7 +73,10 @@ def queryset(self, request, queryset): class DashboardAdmin(ScopedAutocompleteMixin, BaseModelAdmin, ModelAdmin): # A person picking their default board is offered their own. autocomplete_scope = ("owner",) - list_display = ["display_board", "display_owner", "display_default", "item_count"] + # No `default` column. The pointer lives on the user, as + # `default_dashboard`. It is a fact about the owner, not the board. + # It stays on the form, and its filter stays. + list_display = ["display_board", "display_owner", "item_count"] search_fields = ["name", "owner__email"] list_filter = [ ("owner", AutocompleteSelectFilter), From c0012ba172a1faf1572e372af02c22c95ab116c8 Mon Sep 17 00:00:00 2001 From: Thalida Noel Date: Fri, 4 Sep 2026 18:14:57 -0700 Subject: [PATCH 94/94] docs: a recipe description fits on one line `just` prints the comment above a recipe beside its name. Three of them ran past the width of a terminal and wrapped, so the list was harder to read than no list. Each description is now a phrase. What a recipe explains beyond that sits above it, past a blank line, where `just --list` does not reach. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YTrc91QNqcM31iyvkeijEi --- justfile | 53 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/justfile b/justfile index be224c1..a39e1e6 100644 --- a/justfile +++ b/justfile @@ -6,7 +6,9 @@ set dotenv-path := "api/.env.local" # then never share a port or a database. See bin/worktree-env.py. wt := 'eval "$(python3 bin/worktree-env.py)"' -# List the recipes. Keep first: bare `just` runs whatever recipe comes first. +# Keep first: bare `just` runs whatever recipe comes first. + +# List the recipes. default: @just --list --unsorted @@ -14,7 +16,9 @@ default: # that also changes the database. The human asked for exactly this, # since a fresh worktree that only warns still fails unexplained. -# Start Postgres and Redis, migrate, then run the server and the poller. Ctrl-C stops both. +# Ctrl-C stops the server and the poller together. + +# Run the app: stores, migrations, server, poller. [group('Daily')] dev: up migrate @{{wt}} ; trap 'kill 0' EXIT INT TERM ; \ @@ -23,7 +27,9 @@ dev: up migrate uv run celery -A api worker -B -l info & \ uv run python manage.py runserver 0.0.0.0:$DJANGO_PORT -# Run the test suite, host and bin/ scripts together. Arguments pass to both, so `just test -k foo` filters each. +# Arguments pass to both, so `just test -k foo` filters each. + +# Run the test suite, host and bin/ scripts. [group('Daily')] test *args: @{{wt}} ; cd api && uv run pytest -n auto {{args}} @@ -44,7 +50,9 @@ lint: # CVEs and fails the merge on a hit. That needs the network and a # vulnerability database, so it stays out of this recipe. -# Every CI check but the image scan, each where CI runs it. +# Each check runs where CI runs it. + +# Every CI check but the image scan. [group('Daily')] check: image docker compose -f docker-compose.test.yml run --rm pytest -q -n auto --cov-fail-under=85 @@ -56,7 +64,7 @@ check: image -c 'python manage.py makemigrations --check --dry-run' docker compose -f docker-compose.test.yml down -v -# Show this worktree's ports, database and compose project. +# Show this worktree's ports and database. [group('Daily')] info: @{{wt}} ; echo "worktree $WORKTREE_SLUG" ; \ @@ -71,24 +79,26 @@ info: prose *args: @python3 bin/check_prose.py {{args}} -# One-time setup. Every step it calls is runnable on its own. +# Every step it calls is runnable on its own. + +# One-time setup. [group('Lifecycle')] init: env up sync migrate seed pre-commit install @{{wt}} ; cd api && uv run python manage.py collectstatic --noinput -# Start Postgres and Redis for this worktree, waiting until they are healthy. +# Start Postgres and Redis, and wait for both. [group('Lifecycle')] up: @{{wt}} ; docker compose up -d --wait ; \ echo "postgres localhost:$POSTGRES_HOST_PORT redis localhost:$REDIS_HOST_PORT" -# Apply migrations to this worktree's database. +# Apply migrations. [group('Lifecycle')] migrate: @{{wt}} ; cd api && uv run python manage.py migrate -# Drop this worktree's database and build it again from nothing. +# Drop this database and build it again. [group('Lifecycle')] reset: @{{wt}} ; docker compose down -v ; just up ; just migrate ; just seed-live @@ -97,7 +107,7 @@ reset: # project that file uses. Its database is a tmpfs, so it holds nothing. # Add `-v` to drop this checkout's own database too. -# Stop this checkout's containers and remove its networks. +# Stop this checkout's containers. [group('Lifecycle')] clean *args: @{{wt}} ; DIR=$(basename "$PWD") ; \ @@ -110,12 +120,17 @@ clean *args: echo "[just] git branch -d $(git rev-parse --abbrev-ref HEAD)" ; \ fi -# Fetches real status pages, so it needs the network. Fills an empty database: admin, a small catalog, one tracked service. +# Fetches real status pages, so it needs the network. Fills an +# empty database: admin, a small catalog, one tracked service. + +# Fill an empty database with real data. [group('Data')] seed-live: @{{wt}} ; cd api && uv run python manage.py seed_dev -# Probe the recorded status pages and report any that moved or broke. +# Reports any page that moved or broke. + +# Probe the recorded status pages. [group('Data')] probe-pages *args: @{{wt}} ; cd api && uv run python manage.py check_status_pages {{args}} @@ -123,7 +138,9 @@ probe-pages *args: # The tag triggers release.yml: build, sign, smoke test, deploy. A tag # left behind by a failed push is reused rather than duplicated. -# Tag and push a release (v1.2.3, v1.2.3-rc.1). Ships to production. +# Takes v1.2.3 or v1.2.3-rc.1. + +# Tag and push a release. Ships to production. [group('Release')] release VERSION: @set -e ; \ @@ -173,7 +190,7 @@ release VERSION: # Credentials come from api/.env.local. No app argument, so this repo can # only ever deploy its own stack. -# Redeploy the current image without cutting a release. +# Redeploy without cutting a release. [group('Release')] deploy: @set -e ; \ @@ -201,7 +218,9 @@ deploy: # Everything below is a step of `init` or `check`, not a command a # person reaches for. `just ` still runs each one directly. -# Write .env.local, asking for the admin once. Worktrees copy main's. +# Asks for the admin once. A worktree copies main's. + +# Write .env.local. [private] env: @python3 bin/make-env.py @@ -211,7 +230,9 @@ env: sync: cd api && uv sync -# Create the admin in this worktree's database, from .env.local. Never asks. +# Never asks: the values come from .env.local. + +# Create the admin. [private] seed: @{{wt}} ; cd api && uv run python manage.py seed_admin